diff --git a/.agents/skills/local-review-codex/SKILL.md b/.agents/skills/local-review-codex/SKILL.md index cc932f7a4b..ec1277fe3a 100644 --- a/.agents/skills/local-review-codex/SKILL.md +++ b/.agents/skills/local-review-codex/SKILL.md @@ -1,6 +1,6 @@ --- name: local-review-codex -description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy, model, and reasoning effort as the codex-pr-review GitHub action. +description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy and reasoning effort as the codex-pr-review GitHub action, on a newer model. --- # Local Codex Review (pre-push) @@ -11,17 +11,18 @@ before the PR exists. Use this before `git push` on a non-trivial change. **Correspondence with CI** — identical: - Policy: `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test coverage). -- Model: `gpt-5.6-sol`, `model_reasoning_effort="xhigh"`. +- Reasoning effort: `model_reasoning_effort="xhigh"`. - Output: markdown starting with `## Codex Review`, findings tagged P0 / P1 / P2 with file:line. **Differences from CI** — local-only: +- Model is `gpt-6-astra`; CI stays on `gpt-5.6-sol`. Not an oversight to reconcile: `gpt-6-astra` is confirmed on the ChatGPT auth `codex login` uses locally, while CI authenticates with `OPENAI_API_KEY` (`codex-pr-review.yml` prefers it over `CODEX_AUTH_JSON`) and that tier is unverified for the model. Move CI once API access is confirmed, or once CI switches to `CODEX_AUTH_JSON`. - Scope is the current branch vs `main` at the merge-base, **including uncommitted changes** (CI reviews a pushed PR diff). - Sandbox is `read-only` (CI uses `danger-full-access` on an ephemeral runner). Codex reads the diff and files but cannot modify your working tree. - Fresh context is inherent: `codex exec` is a separate cold process, so it does not anchor on the current chat session — the same reason `local-review` insists on a subagent. ## Prerequisites -- `codex` CLI **>= 0.144.1** installed and authed (`codex login` or `OPENAI_API_KEY`). Older CLIs reject `gpt-5.6-sol` with "requires a newer version of Codex". Upgrade with `npm install --global @openai/codex@0.144.1` (may need `sudo` for a global install). Keep this in sync with the pin in `.github/workflows/codex-pr-review.yml`. +- `codex` CLI **>= 0.153.4** installed and authed via `codex login` (an `OPENAI_API_KEY` in the environment takes priority and may not reach `gpt-6-astra` — see the model note above). Older CLIs reject the model with "requires a newer version of Codex"; `run.sh` checks the version up front. Upgrade with `npm install --global @openai/codex@0.153.4` (may need `sudo` for a global install). This matches the pin in `.github/workflows/codex-pr-review.yml` — the CLI version is the same on both sides, only the model differs. - `git fetch` the base ref if it's stale, so the merge-base is accurate. ## Run diff --git a/.agents/skills/local-review-codex/run.sh b/.agents/skills/local-review-codex/run.sh index d6491099c2..948d3820eb 100755 --- a/.agents/skills/local-review-codex/run.sh +++ b/.agents/skills/local-review-codex/run.sh @@ -1,21 +1,44 @@ #!/usr/bin/env bash # Local Codex review — mirrors the .github/workflows/codex-pr-review.yml CI job, # but scoped to this branch's unpushed work (committed + uncommitted) so you can -# review before pushing. Same policy (REVIEW.md), same model (gpt-5.6-sol) and -# reasoning effort (xhigh) as CI. Runs read-only: Codex cannot modify your tree. +# review before pushing. Same policy (REVIEW.md) and reasoning effort (xhigh) as CI. +# +# The model deliberately differs from CI: gpt-6-astra is confirmed available on the +# ChatGPT auth `codex login` uses here, but CI authenticates with OPENAI_API_KEY and +# that tier is unverified for it, so codex-pr-review.yml stays on gpt-5.6-sol. # # Usage: run.sh [BASE_REF] (BASE_REF defaults to "main") set -euo pipefail +MODEL="gpt-6-astra" +CODEX_MIN="0.153.4" + BASE_REF="${1:-main}" REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" if ! command -v codex >/dev/null 2>&1; then - echo "codex CLI not found. Install with: npm install --global @openai/codex@0.144.1" >&2 + echo "codex CLI not found. Install with: npm install --global @openai/codex@$CODEX_MIN" >&2 exit 1 fi +# Older CLIs reject the model with an error that never names the CLI version as the +# cause, so check it up front rather than letting the exec fail opaquely. The `|| true` +# keeps an unrecognised --version format from aborting under `set -e`: an unparseable +# version means "cannot tell", which must fall through to the exec, not kill the review. +CODEX_VER="$(codex --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" +if [ -n "$CODEX_VER" ] && [ "$(printf '%s\n%s\n' "$CODEX_MIN" "$CODEX_VER" | sort -V | head -1)" != "$CODEX_MIN" ]; then + echo "codex $CODEX_VER is too old for $MODEL (need >= $CODEX_MIN). Upgrade with: npm install --global @openai/codex@$CODEX_MIN" >&2 + exit 1 +fi + +# codex prefers OPENAI_API_KEY over the ChatGPT credentials `codex login` stores, and +# that tier is not confirmed for $MODEL — the resulting failure names the model, not the +# auth that selected it. +if [ -n "${OPENAI_API_KEY:-}" ]; then + echo "warning: OPENAI_API_KEY is set and takes priority over 'codex login' credentials; $MODEL may be unavailable on that tier." >&2 +fi + # Resolve the base to a concrete commit, preferring a local ref but falling back to # the remote-tracking ref — checkouts (CI, single-branch clones) often have only # origin/main, not a local main. @@ -80,7 +103,7 @@ EOF codex exec \ -C "$REPO_ROOT" \ - -m gpt-5.6-sol \ + -m "$MODEL" \ -c 'model_reasoning_effort="xhigh"' \ -s read-only \ -o "$OUT" \ diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md index 6a72949c96..f2a9ec8dad 100644 --- a/.agents/skills/pr/SKILL.md +++ b/.agents/skills/pr/SKILL.md @@ -210,7 +210,7 @@ the diff first — `git diff --name-only main...HEAD` answers most of these. **Flip without asking** when it is self-contained: a single-file fix, test-only, docs-only, one call site, no new public surface. -Unattended (webmux oneshot) there is nobody to ask, so the judgement holds and the action +Unattended (a one-shot run) there is nobody to ask, so the judgement holds and the action degrades: flip the self-contained ones, and leave the rest at a clean draft with a line in the PR description saying why — `left in draft: adds a migration, wants a human look before ready`. Don't flip a wide-blast-radius change just because the round came back clean, and don't ask a diff --git a/.claude/settings.json b/.claude/settings.json index a464ca3719..37aaeeb0f5 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -55,22 +55,18 @@ "mcp__claude_ai_Gmail__list_drafts" ], "deny": [ - "Read(.env)", - "Read(.env.*)", - "Read(**/.env)", - "Read(**/.env.*)", - "Read(**/secrets/**)", - "Read(**/*.pem)", - "Read(**/*.key)", - "Read(**/credentials.json)", - "Read(**/.secret*)", - "Read(**/.secrets*)", - "Read(**/*.secret)", - "Read(**/*.secrets)", "Edit(.env)", "Edit(.env.*)", "Edit(**/.env)", - "Edit(**/.env.*)" + "Edit(**/.env.*)", + "Edit(**/secrets/**)", + "Edit(**/*.pem)", + "Edit(**/*.key)", + "Edit(**/credentials.json)", + "Edit(**/.secret*)", + "Edit(**/.secrets*)", + "Edit(**/*.secret)", + "Edit(**/*.secrets)" ], "ask": [ "Bash(rmdir:*)", diff --git a/.github/actions/sign-attest-image/action.yml b/.github/actions/sign-attest-image/action.yml new file mode 100644 index 0000000000..de0133dac0 --- /dev/null +++ b/.github/actions/sign-attest-image/action.yml @@ -0,0 +1,56 @@ +name: Sign image and attach provenance +description: > + Keyless-signs a pushed image digest with cosign (index and per-arch + manifests) and records SLSA provenance as a GitHub artifact attestation + pushed to the registry. SBOMs are not generated here: the build step embeds + them as BuildKit attestation manifests (depot `sbom: true`), which the index + signature then covers. The calling job must already be logged in to the + registry and must have id-token: write, attestations: write and + packages: write permissions (write-all covers all three). +inputs: + image: + description: "Fully-qualified image name without tag, e.g. ghcr.io/windmill-labs/windmill" + required: true + digest: + description: "Pushed manifest digest (sha256:...) from build-push-action" + required: true +runs: + using: composite + steps: + - name: Preflight + shell: bash + env: + DIGEST: ${{ inputs.digest }} + run: | + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then + echo "::error::No OIDC token available; the calling job needs id-token: write" + exit 1 + fi + case "$DIGEST" in + sha256:*) ;; + *) + echo "::error::digest '$DIGEST' is not a sha256: digest" + exit 1 + ;; + esac + + # cosign v2 writes the classic sha256-.sig tag format that the + # installed base of cosign clients can verify; v3's bundle format cannot be + # verified by v2 clients yet, so stay on v2 until v3 verification is common. + - uses: sigstore/cosign-installer@v4.1.2 + with: + cosign-release: "v2.6.5" + + - name: Cosign keyless sign (index + per-arch manifests) + shell: bash + env: + IMAGE: ${{ inputs.image }} + DIGEST: ${{ inputs.digest }} + run: cosign sign --yes --recursive "${IMAGE}@${DIGEST}" + + - name: SLSA provenance (GitHub artifact attestation) + uses: actions/attest-build-provenance@v4 + with: + subject-name: ${{ inputs.image }} + subject-digest: ${{ inputs.digest }} + push-to-registry: true diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2f23b21eb8..72bb2b9d8c 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -12,6 +12,7 @@ sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath} sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json +sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml diff --git a/.github/change-versions.sh b/.github/change-versions.sh index ffb73f6180..1772ddbc89 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -13,6 +13,7 @@ sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/o sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/jsr.json +sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/windmill-yaml-validator/package.json sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml @@ -33,3 +34,5 @@ cd ${root_dirpath}/frontend && npm i --package-lock-only --ignore-scripts # The CLI installs this package on every `bun install`, which would otherwise rewrite the # lockfile's version and leave a dirty tree. cd ${root_dirpath}/windmill-yaml-validator && npm i --package-lock-only --ignore-scripts + +cd ${root_dirpath}/chat-sdk && npm i --package-lock-only --ignore-scripts diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 80970d1e76..0d174a548f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,4 +1,13 @@ -# Basic set up for three package managers +# Weekly, grouped version updates for every shipped manifest, plus one grouped +# security-update PR per manifest. Security-update PRs have no open-PR limit, so +# without an `applies-to: security-updates` group each alert would get its own PR. +# Bun has no security updates: /cli alerts are fixed through its version group. +# +# Deliberately NOT covered here: +# - /rust-client: Cargo.toml is generated and gitignored +# - /backend/parsers/windmill-parser-wasm: excluded from the workspace, needs nightly cargo +# - /benchmarks/pulumi: one-off load-test infrastructure, not shipped +# - /integration_tests: CI-only version: 2 updates: @@ -13,12 +22,32 @@ updates: directory: "/frontend" schedule: interval: "weekly" + groups: + # minor + patch bumps in one PR; majors in their own PR so they can be + # reviewed separately without blocking the routine bumps + frontend-deps: + patterns: ["*"] + update-types: ["minor", "patch"] + frontend-majors: + patterns: ["*"] + update-types: ["major"] + frontend-security: + applies-to: security-updates + patterns: ["*"] # Maintain dependencies for cargo - package-ecosystem: "cargo" directory: "/backend" schedule: interval: "weekly" + groups: + # majors stay as individual PRs: they usually need code changes + cargo-deps: + patterns: ["*"] + update-types: ["minor", "patch"] + cargo-security: + applies-to: security-updates + patterns: ["*"] # Maintain dependencies for Docker - package-ecosystem: "docker" @@ -31,3 +60,52 @@ updates: directory: "/python-client/wmill" schedule: interval: "weekly" + groups: + python-client-security: + applies-to: security-updates + patterns: ["*"] + + # Maintain dependencies for the CLI (bun.lock) + - package-ecosystem: "bun" + directory: "/cli" + schedule: + interval: "weekly" + groups: + cli-deps: + patterns: ["*"] + + # Maintain dependencies for the YAML validator + - package-ecosystem: "npm" + directory: "/windmill-yaml-validator" + schedule: + interval: "weekly" + groups: + yaml-validator-deps: + patterns: ["*"] + yaml-validator-security: + applies-to: security-updates + patterns: ["*"] + + # Maintain dependencies for the TypeScript client (devDependencies only) + - package-ecosystem: "npm" + directory: "/typescript-client" + schedule: + interval: "weekly" + groups: + typescript-client-deps: + patterns: ["*"] + typescript-client-security: + applies-to: security-updates + patterns: ["*"] + + # Maintain dependencies for multiplayer + - package-ecosystem: "npm" + directory: "/multiplayer" + schedule: + interval: "weekly" + groups: + multiplayer-deps: + patterns: ["*"] + multiplayer-security: + applies-to: security-updates + patterns: ["*"] diff --git a/.github/workflows/ai-agent-tests.yml b/.github/workflows/ai-agent-tests.yml index cd07602109..8a3e9e6dbc 100644 --- a/.github/workflows/ai-agent-tests.yml +++ b/.github/workflows/ai-agent-tests.yml @@ -63,11 +63,11 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.11" diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 87eefab0f3..573bc77df9 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -77,7 +77,7 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: # Node 24 ships npm 11, which frontend/package-lock.json is authored # with; npm 10 rejects it ("Missing: picomatch@4.0.5 from lock file"). diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index e2e7318ab6..54a62ec495 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -67,7 +67,7 @@ jobs: # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v7 with: go-version: 1.21.5 @@ -75,11 +75,11 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" - - uses: astral-sh/setup-uv@v6.2.1 + - uses: astral-sh/setup-uv@v7 with: version: "0.11.24" @@ -101,7 +101,6 @@ jobs: - name: Install OpenSSL via vcpkg run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cdd0e2a212..fea84e7781 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -53,16 +53,16 @@ jobs: # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v7 with: go-version: 1.21.5 - uses: oven-sh/setup-bun@v2 with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: - node-version: "20" - - uses: astral-sh/setup-uv@v6.2.1 + node-version: "24" + - uses: astral-sh/setup-uv@v7 with: version: "0.11.24" - uses: shivammathur/setup-php@v2 diff --git a/.github/workflows/build_cli_image.yml b/.github/workflows/build_cli_image.yml index 5b71a786aa..de64a76583 100644 --- a/.github/workflows/build_cli_image.yml +++ b/.github/workflows/build_cli_image.yml @@ -13,9 +13,13 @@ permissions: contents: read id-token: write packages: write + attestations: write jobs: publish_cli: + # a tag-targeted dispatch would republish the release tags unsigned, + # un-verifying the release; to republish a release, re-push its tag + if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud steps: - uses: actions/checkout@v4 @@ -42,14 +46,23 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: file: "./docker/DockerfileCli" platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ steps.meta.outputs.tags }} labels: | ${{ steps.meta.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 84f4366235..46befe42b6 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -62,7 +62,6 @@ jobs: - name: Cargo build binary windows timeout-minutes: 180 run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 diff --git a/.github/workflows/check-docs-links.yml b/.github/workflows/check-docs-links.yml index ec17c06771..e32e976f0c 100644 --- a/.github/workflows/check-docs-links.yml +++ b/.github/workflows/check-docs-links.yml @@ -16,7 +16,7 @@ jobs: sparse-checkout: | frontend/src .github/scripts - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "22.x" - name: Verify docs links are not 404 diff --git a/.github/workflows/check-system-prompts.yml b/.github/workflows/check-system-prompts.yml index 3e684c82f9..370a177716 100644 --- a/.github/workflows/check-system-prompts.yml +++ b/.github/workflows/check-system-prompts.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index e26231eda4..19afa777ae 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" @@ -89,7 +89,7 @@ jobs: cache-workspaces: backend - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" @@ -143,7 +143,7 @@ jobs: cache-workspaces: backend - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 26b2d9aae8..354e8a60ca 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -213,13 +213,13 @@ jobs: - name: Set up Node.js if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22 - name: Install Codex CLI if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - run: npm install --global @openai/codex@0.144.1 + run: npm install --global @openai/codex@0.153.4 - name: Configure Codex auth if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 433f3a86b3..a8b20623bc 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -86,11 +86,13 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} build-args: | features=ce WM_BUILD_VERSION=${{ github.sha }} @@ -100,6 +102,13 @@ jobs: labels: | ${{ steps.meta-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} + build_ee: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || github.event.inputs.ee @@ -149,11 +158,13 @@ jobs: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} build-args: | features=ee WM_BUILD_VERSION=${{ github.sha }} @@ -164,6 +175,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee + digest: ${{ steps.docker_build.outputs.digest }} + attach_amd64_binary_to_release: needs: [build, build_ee] runs-on: ubicloud @@ -358,6 +376,21 @@ jobs: docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main + - uses: sigstore/cosign-installer@v4.1.2 + if: startsWith(github.ref, 'refs/tags/v') + with: + cosign-release: "v2.6.5" + # end-to-end release guard: the version tag pushed by this run must + # verify against this exact run's identity (the mutable :latest/:dev + # tags race with concurrent main builds, so they are not asserted here) + - name: Verify release image is signed + if: startsWith(github.ref, 'refs/tags/v') + run: | + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/windmill-labs/windmill/.github/workflows/docker-image.yml@${GITHUB_REF}" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF_NAME#v}" + tag_latest_ee: runs-on: ubicloud needs: [run_integration_test, build_ee] @@ -379,6 +412,21 @@ jobs: docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main + - uses: sigstore/cosign-installer@v4.1.2 + if: startsWith(github.ref, 'refs/tags/v') + with: + cosign-release: "v2.6.5" + # end-to-end release guard: the version tag pushed by this run must + # verify against this exact run's identity (the mutable :latest/:dev + # tags race with concurrent main builds, so they are not asserted here) + - name: Verify release ee image is signed + if: startsWith(github.ref, 'refs/tags/v') + run: | + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/windmill-labs/windmill/.github/workflows/docker-image.yml@${GITHUB_REF}" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${GITHUB_REF_NAME#v}" + verify_ee_image_vulnerabilities: runs-on: ubicloud needs: [tag_latest_ee] @@ -493,11 +541,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileCuda" tags: | ${{ steps.meta-ee-public.outputs.tags }} @@ -505,6 +555,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda + digest: ${{ steps.docker_build.outputs.digest }} + build_slim: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] @@ -537,17 +594,26 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlim" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | ${{ steps.meta-ee-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim + digest: ${{ steps.docker_build.outputs.digest }} + build_ee_slim: needs: [build_ee] runs-on: ubicloud @@ -582,11 +648,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlimEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} @@ -594,6 +662,13 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim + digest: ${{ steps.docker_build.outputs.digest }} + build_full: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] @@ -626,17 +701,26 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFull" tags: | ${{ steps.meta-public.outputs.tags }} labels: | ${{ steps.meta-public.outputs.labels }} + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-full + digest: ${{ steps.docker_build.outputs.digest }} + build_ee_full: if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] @@ -669,14 +753,23 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly ee + id: docker_build uses: depot/build-push-action@v1 with: context: . platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFullEe" tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/frontend-check.yml b/.github/workflows/frontend-check.yml index a675c17444..6347614e16 100644 --- a/.github/workflows/frontend-check.yml +++ b/.github/workflows/frontend-check.yml @@ -9,6 +9,8 @@ on: push: paths: - "frontend/**" + # The flow chat compiles the chat SDK's source in (svelte.config.js alias). + - "chat-sdk/src/**" - ".github/workflows/frontend-check.yml" jobs: @@ -16,7 +18,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@v7 with: node-version: 24 cache: "npm" diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index ee732569dd..b19f899a20 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -9,6 +9,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -22,6 +23,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -59,7 +61,7 @@ jobs: echo "$CHANGED_FILES" # Direct git sync file changes — always relevant. - if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|frontend/src/lib/hubPaths\.json|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Relevant: direct git sync file changes" exit 0 @@ -139,7 +141,7 @@ jobs: with: deno-version: v2.x - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/go_on_release.yml b/.github/workflows/go_on_release.yml index 78ce12c6c3..6e53db09e7 100644 --- a/.github/workflows/go_on_release.yml +++ b/.github/workflows/go_on_release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v7 - name: install_jq run: | sudo apt-get update diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index a41bd80854..631ab53153 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -10,18 +10,29 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" - run: cd typescript-client && ./publish.sh --access public && cd .. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + publish_chat_sdk: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v7 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + - run: cd chat-sdk && npm ci && npm run build && npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index 9b536b6ed5..a89275cfb5 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -206,7 +206,7 @@ jobs: - name: Set up Node.js if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22 diff --git a/.github/workflows/publish-cli-docs.yml b/.github/workflows/publish-cli-docs.yml index 9e76117eb5..d6a2bdb332 100644 --- a/.github/workflows/publish-cli-docs.yml +++ b/.github/workflows/publish-cli-docs.yml @@ -38,7 +38,7 @@ jobs: ssh-key: ${{ secrets.CLI_DOCS_DEPLOY_KEY }} fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/publish_extra.yml b/.github/workflows/publish_extra.yml index 49964c95d9..ca62933292 100644 --- a/.github/workflows/publish_extra.yml +++ b/.github/workflows/publish_extra.yml @@ -84,6 +84,9 @@ jobs: publish_extra: needs: [sleep, test_extra] + # a tag-targeted dispatch would republish the release tags unsigned, + # un-verifying the release; to republish a release, re-push its tag + if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 @@ -112,15 +115,24 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push publicly + id: docker_build uses: depot/build-push-action@v1 with: context: . file: ./docker/DockerfileExtra platforms: linux/amd64,linux/arm64 push: true + sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ steps.meta.outputs.tags }} labels: | ${{ steps.meta.outputs.labels }} org.opencontainers.image.licenses=AGPLv3 + + - name: Sign and attest release image + if: startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' + uses: ./.github/actions/sign-attest-image + with: + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + digest: ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index d159619fba..b94d5d57c7 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -51,7 +51,6 @@ jobs: - name: Cargo build windows timeout-minutes: 180 run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index efcbcd6ec1..260fbb3a02 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -32,6 +32,25 @@ jobs: working-directory: ./typescript-client run: bun test --timeout 120000 tests/ + chat-sdk: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - uses: actions/setup-node@v7 + with: + node-version: "20.x" + + - name: Run tests + working-directory: ./chat-sdk + run: npm ci && npm run check && bun test + python-client: runs-on: ubuntu-latest steps: @@ -39,7 +58,7 @@ jobs: uses: actions/checkout@v4 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 # The interpreter is named explicitly: on a clean checkout uv picks the # runner's system python and stops with "not compatible with the locked diff --git a/.github/workflows/validate-openapi.yml b/.github/workflows/validate-openapi.yml index 5bc59ebc4c..846621b503 100644 --- a/.github/workflows/validate-openapi.yml +++ b/.github/workflows/validate-openapi.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: '20' diff --git a/.github/workflows/yaml-validator-tests.yml b/.github/workflows/yaml-validator-tests.yml index 04d31451d4..c5524312cb 100644 --- a/.github/workflows/yaml-validator-tests.yml +++ b/.github/workflows/yaml-validator-tests.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 2c5c3efad9..438887d2eb 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.798.1" + ".": "1.816.0" } diff --git a/AGENTS.md b/AGENTS.md index 47919cfeda..77b38c3267 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,14 +30,18 @@ Open-source platform for internal tools, workflows, API integrations, background reaches the DB only through the API, so `Connection::Http` paths are never taken by a plain `cargo run`; a normal build cannot start one at all. - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow +- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation + scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and + that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates + users, tokens or sessions. - **Product telemetry**: `docs/feature-telemetry.md` — when to instrument a new feature with `feature_usage`, and the four-step recipe. An unregistered `(feature, kind)` pair is dropped silently, so frontend-only instrumentation records nothing. - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead. -- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy, `gpt-5.6-sol`, `xhigh` reasoning; requires the `codex` CLI >= 0.144.1. -- **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` +- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy and `xhigh` reasoning, on `gpt-6-astra` rather than the action's `gpt-5.6-sol`; requires the `codex` CLI >= 0.153.4. +- **Domain guides**: `.claude/skills/native-trigger/` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. @@ -51,12 +55,23 @@ Open-source platform for internal tools, workflows, API integrations, background > defaults in this section apply only to a plain single checkout. **Discover the real > values before running anything** — see "Per-worktree ports and database" below. -**Check whether they are already running before starting anything.** In a webmux worktree -(`$WEBMUX_WORKTREE_PATH` is set) the backend and frontend are already up in sibling tmux panes — -use those, don't spawn your own. `tmux list-panes -t "$(tmux display-message -p -t "$TMUX_PANE" -'#{window_id}')" -F '#{pane_index} #{pane_current_command}'` shows what is running; read its log -with `tmux capture-pane`, and see `backend/AGENTS.md` to restart it with different cargo features. -A second server started in your own shell fights the first one for the port. The commands below +**Check whether they are already running before starting anything.** In a managed worktree the +backend and frontend are already up in sibling panes — use those, don't spawn your own. A second +server started in your own shell fights the first one for the port. + +**herdr** is the worktree manager; `$HERDR_ENV=1` marks one of its panes. `herdr pane list +--workspace "$HERDR_WORKSPACE_ID"` lists this worktree's panes — match on `cwd`, since the servers +run from `backend/` and `frontend/` — and `herdr pane read --source recent-unwrapped +--lines 50` reads one's log; `herdr --skill` prints the full reference for +inspecting and driving panes and agents. The plugins that provision these worktrees live in the +private `windmill-labs/windmill-herdr` — clone it and run `./setup.sh` to install them and the +keybindings they need. + +A worktree from the older **webmux** setup sets `$WEBMUX_WORKTREE_PATH` instead and is driven +through tmux: `tmux list-panes -t "$(tmux display-message -p -t "$TMUX_PANE" '#{window_id}')" -F +'#{pane_index} #{pane_current_command}'` for what is running, `tmux capture-pane` for a pane's log. + +See `backend/AGENTS.md` to restart the backend with different cargo features. The commands below are for a plain checkout with nothing running. - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) @@ -68,10 +83,16 @@ are for a plain checkout with nothing running. ### Per-worktree ports and database -In a webmux worktree the authoritative values live in -`$(git rev-parse --git-dir)/webmux/runtime.env` — `BACKEND_PORT`, `FRONTEND_PORT`, -`DATABASE_URL`, `CARGO_FEATURES`, `WM_DB_NAME`. Every pane sources it at startup. Read that -first: it is not a `.env*` file, so the repo's secret-file read rules don't stand in the way. +**`.env.local` in the worktree root holds the real values** — `BACKEND_PORT`, `FRONTEND_PORT`, +`REMOTE`, `DATABASE_URL`, `CARGO_FEATURES`, `WM_DB_NAME`. Every manager writes those fields, so it +is correct whichever one you are in. Read it with `cat .env.local` from a shell: the file-read +tool denies every `.env.*` path, and `DATABASE_URL` is written with an `export` prefix that a +`^DATABASE_URL=` grep misses. + +herdr keeps nothing besides that file; its hooks write it and read it back. A webmux worktree +additionally has `$(git rev-parse --git-dir)/webmux/runtime.env`, which every pane sources at +startup and which carries the extras `.env.local` lacks: `WEBMUX_*`, `WM_CLONE_DB`, +`USE_RUST_PLUGIN`. In a plain checkout, fall back to `.env` / `.env.local` (repo root) and `backend/.env`. @@ -80,19 +101,25 @@ hook. It is not a copy of the main dev instance: you get the `admins` workspace, `admin@windmill.dev` superadmin, the license key copied from the base database, and whatever the migrations seed — and none of your own workspaces, scripts, flows or apps. Create whatever a test needs. Cloning the base `windmill` database instead is -opt-in per project via `WM_CLONE_DB` in `.webmux.yaml`; read the note there before turning it on. +opt-in via `WM_CLONE_DB` (the `windmill.worktree` plugin's `config.env` under herdr, or +`.webmux.yaml` under webmux); read the note in `.webmux.yaml` before turning it on. The database is named after the **worktree directory, not the branch** (`scripts/worktree-common.sh`): `windmill_` + the directory basename with `-` → `_`, which Postgres then truncates at 63 -characters. Branch `hugo/win-2340-ai-agent-evals-standalone-agent-runs-and-eval-datasets` sits in -a worktree directory named `win-2340-…`, so its database is -`windmill_win_2340_ai_agent_evals_standalone_agent_runs_and_eval` — no `hugo_`, and the tail -chopped. Take `WM_DB_NAME` from `runtime.env` instead of reconstructing the name. Read those, or -discover from what is already running: +characters. herdr creates the directory under `~/.herdr/worktrees//` from the branch name +with each `/` turned into `-`, so branch +`hugo/win-2544-add-options-field-to-the-postgresql-resource-type` gets the database +`windmill_hugo_win_2544_add_options_field_to_the_postgresql_reso` — the whole branch name, cut +mid-word at the limit. The two drift apart as soon as the branch is renamed, and webmux named its +directories after the ticket alone, so reconstructing the name from the branch you are on is +wrong in both. Take `WM_DB_NAME` from `.env.local` instead. Read that, or discover from what is +already running: ```bash +# match on the worktree directory, truncated the way Postgres truncates it (63 - len('windmill_')): psql postgres://postgres:changeme@localhost:5432/postgres -tAc \ - "select datname from pg_database where datname like 'windmill%'" | grep "$(git branch --show-current | tr - _)" + "select datname from pg_database where datname like 'windmill%'" \ + | grep -F "$(basename "$(git rev-parse --show-toplevel)" | tr '/-' '_' | cut -c1-54)" # the port the frontend actually proxies to (REMOTE of this worktree's vite): for p in $(pgrep -f vite); do case "$(readlink /proc/$p/cwd)" in *"$(basename "$(git rev-parse --show-toplevel)")"*) tr '\0' '\n' < /proc/$p/environ | grep -E '^REMOTE=|^PORT=';; esac; done @@ -165,6 +192,15 @@ $NAV --root backend callees "X" # what does X call? - Search for existing code to reuse before writing new code - Follow established patterns in the codebase - Keep changes focused — don't refactor beyond what's asked +- **A simpler design found late is still the design.** Work already spent is not an argument + for a shape, and neither is a clean review round, a passing suite, or a long PR thread. The + signal to stop and re-derive rather than patch again is a change that keeps growing to defend + its own structure: each review finding fixing an assumption the previous fix broke, the same + class of bug reappearing somewhere new, or most of the diff being consequences of one early + choice rather than the thing you set out to do. When that happens, say plainly what the + simpler design is and what switching costs — a migration, a review cycle restarted from zero, + work discarded — and let the user decide. Do not keep paying down the harder one because it + is nearly finished, and do not present the accumulated cost as a reason to continue. - **Ship only the tests the PR needs.** A committed test must pin behavior a future change could plausibly break, and be the smallest setup that exercises the new logic. While developing, write as many exhaustive tests and do as much manual testing as you need to convince yourself the change works — then remove that scaffolding before marking the PR ready, keeping only the essential regression guard(s). A test that merely re-exercises pre-existing behavior, or needs elaborate fixtures to assert something trivial, is scaffolding: delete it. If nothing meaningful is left to guard, ship no test rather than a ceremonial one. - **Comments record constraints, not narration.** Write a comment only for what the code can't show: why a non-obvious approach is required, what breaks if it's "simplified" away. State each invariant once, at the place where someone would break it, in ≤4 lines. Don't describe what the next line does, don't repeat the same rationale at multiple sites, and don't address the PR reviewer (justifying a change belongs in the PR description, not the code). Reference nothing ephemeral — no numbered steps from your dev flow, no "the poller / the test does X" scaffolding, no transient state that won't exist for the next reader; keep only the essential, durable rationale. Describe the code as it is, never its drafting history: "we no longer do X", "unchanged behavior", "instead of the previous approach" are meaningless to a reader who never saw the earlier iteration — before finishing, reread your comments as if the current state is the only state that ever existed. - **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b042bab5..e7e039fb45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,454 @@ # Changelog +## [1.816.0](https://github.com/windmill-labs/windmill/compare/v1.815.0...v1.816.0) (2026-09-21) + + +### Features + +* add an ACL editor for data table roles ([#11063](https://github.com/windmill-labs/windmill/issues/11063)) ([9994e03](https://github.com/windmill-labs/windmill/commit/9994e03bc75404503001a5db37e175b66e7371f2)) +* add per-model context window override in workspace AI settings ([#11232](https://github.com/windmill-labs/windmill/issues/11232)) ([c637b06](https://github.com/windmill-labs/windmill/commit/c637b06a548a5725aceb7504bcb62383d35614ac)) +* **ai-chat:** disclose an app's exposure on deploy, refuse drag-and-drop apps ([#11137](https://github.com/windmill-labs/windmill/issues/11137)) ([85b36a2](https://github.com/windmill-labs/windmill/commit/85b36a2de5f3fa08750b6d2ab8203f4b745cfdd2)) +* data table roles in the DB manager and raw apps ([#11139](https://github.com/windmill-labs/windmill/issues/11139)) ([e1e3692](https://github.com/windmill-labs/windmill/commit/e1e3692fbc82d50c021ef8bf8ca7019d760705f0)) +* drafts follow their item through a move; behind means base ≠ head ([#10577](https://github.com/windmill-labs/windmill/issues/10577)) ([f00b2fc](https://github.com/windmill-labs/windmill/commit/f00b2fcb1e65cb8f8e95d55f7cc58b816ac7d89d)) +* edit variables, resources and triggers in their own session tab ([#11206](https://github.com/windmill-labs/windmill/issues/11206)) ([6186a06](https://github.com/windmill-labs/windmill/commit/6186a0645d3f3e92e06b666a6b5774b8af2b488f)) +* let every EE instance set the announcement banner ([#11238](https://github.com/windmill-labs/windmill/issues/11238)) ([87761d1](https://github.com/windmill-labs/windmill/commit/87761d179799026ebaf9ed91bf589852b5453ee3)) +* snap-collapse the sidebar by dragging it to the left edge ([#11225](https://github.com/windmill-labs/windmill/issues/11225)) ([56a6ad3](https://github.com/windmill-labs/windmill/commit/56a6ad322e8354695a0a96eef9e83ffa135b3ad4)) + + +### Bug Fixes + +* bump git sync hub scripts to cli 1.815.1 ([#11239](https://github.com/windmill-labs/windmill/issues/11239)) ([9e0e835](https://github.com/windmill-labs/windmill/commit/9e0e835c6876305d3a44902fb1432fd9c2348a7c)) +* check custom_instance_user can connect during instance database setup ([#11244](https://github.com/windmill-labs/windmill/issues/11244)) ([1b0f119](https://github.com/windmill-labs/windmill/commit/1b0f119c7b8ac2aacf2e2b4428c57f661df87ae5)) +* **cli:** read wmill.yaml from the branch a git-sync deploy writes to ([#11236](https://github.com/windmill-labs/windmill/issues/11236)) ([974b026](https://github.com/windmill-labs/windmill/commit/974b0265f85f9cfac65bbd2f6ebbcb0e966e3301)) +* guard data table migration routes against operators and unauthorized authors ([#11243](https://github.com/windmill-labs/windmill/issues/11243)) ([b742544](https://github.com/windmill-labs/windmill/commit/b7425443e1e08be029fc676047f727f84b646323)) +* tell the user when a draft conflict has stopped their edits saving ([#11241](https://github.com/windmill-labs/windmill/issues/11241)) ([d8ebbc0](https://github.com/windmill-labs/windmill/commit/d8ebbc06f5c56dad2570115700d640105d856c57)) + + +### Performance Improvements + +* index audit logs by workspace and operation ([#11235](https://github.com/windmill-labs/windmill/issues/11235)) ([3b8c326](https://github.com/windmill-labs/windmill/commit/3b8c3264ecb33884420b1bdac9cae9c841e721a0)) +* keep the sessions page fast with many ai sessions ([#11224](https://github.com/windmill-labs/windmill/issues/11224)) ([4cbc944](https://github.com/windmill-labs/windmill/commit/4cbc944c0bcb938300c964437d745d15a3ea608f)) +* retire the legacy pre-partitioning audit table ([#11240](https://github.com/windmill-labs/windmill/issues/11240)) ([9f2685d](https://github.com/windmill-labs/windmill/commit/9f2685d788e624c9ed9db5cbaee456ea21af2b29)) + +## [1.815.0](https://github.com/windmill-labs/windmill/compare/v1.814.0...v1.815.0) (2026-09-18) + + +### Features + +* add a workspace toggle that adds its admins and developers to new forks ([#11215](https://github.com/windmill-labs/windmill/issues/11215)) ([9d335de](https://github.com/windmill-labs/windmill/commit/9d335de87a4dbaa51038d55afe8d980761dcdfaf)) +* add an instance setting to refuse a token in MCP URLs ([#11162](https://github.com/windmill-labs/windmill/issues/11162)) ([37e493a](https://github.com/windmill-labs/windmill/commit/37e493ae66ed5c000ecac492d60fc0fdf4bda71f)) +* **ai-chat:** render get_run through the run tools' card ([#11204](https://github.com/windmill-labs/windmill/issues/11204)) ([6f9c4dc](https://github.com/windmill-labs/windmill/commit/6f9c4dc29455d13b0e64af05c2d6aa8bd5ff4fd6)) +* **ai-chat:** tell the chat which kind of app it is looking at ([#11208](https://github.com/windmill-labs/windmill/issues/11208)) ([c8c06d8](https://github.com/windmill-labs/windmill/commit/c8c06d8f79774abf109192e71a8b6fc37c7937ba)) +* attach files to a flow chat message ([#11185](https://github.com/windmill-labs/windmill/issues/11185)) ([c4c9677](https://github.com/windmill-labs/windmill/commit/c4c9677982b75c63d98ebf85b1904e0c341ba957)) +* **auth:** 2 h login links and a click-to-sign-in page for emailed ones ([#11203](https://github.com/windmill-labs/windmill/issues/11203)) ([5639187](https://github.com/windmill-labs/windmill/commit/5639187fec6d517a72e82df49d63d7438301127c)) +* cap user token expiration with an instance setting ([#11159](https://github.com/windmill-labs/windmill/issues/11159)) ([9320312](https://github.com/windmill-labs/windmill/commit/9320312eac56f944c4d31504601293ab4e816ccc)) +* flow chat job-backed detail, smooth streaming and answer chrome ([#11186](https://github.com/windmill-labs/windmill/issues/11186)) ([e2a91ca](https://github.com/windmill-labs/windmill/commit/e2a91ca2b141a0bee64ddfafe3cbf3ff86b6fb35)) +* put a data table's connection under Postgres roles ([#11020](https://github.com/windmill-labs/windmill/issues/11020)) ([0e807fb](https://github.com/windmill-labs/windmill/commit/0e807fb1dd80d7536ec144cd49445abc7961e504)) +* render an AI agent result as its answer, not as raw JSON ([#11051](https://github.com/windmill-labs/windmill/issues/11051)) ([a089928](https://github.com/windmill-labs/windmill/commit/a08992834d45d0211336f4fc32c3421646ca47c5)) +* support $flow_expr[...] dynamic tags on flow steps ([#11170](https://github.com/windmill-labs/windmill/issues/11170)) ([48f0025](https://github.com/windmill-labs/windmill/commit/48f00259c5e7361d3553dbcb809461e1cde96f8f)) + + +### Bug Fixes + +* **frontend:** inline only the package version, not the whole package.json ([#11191](https://github.com/windmill-labs/windmill/issues/11191)) ([813e486](https://github.com/windmill-labs/windmill/commit/813e486e166ac6215364817a7732b66f8dc1d463)) +* keep instance groups when editing auto-invite ([#11217](https://github.com/windmill-labs/windmill/issues/11217)) ([df61dea](https://github.com/windmill-labs/windmill/commit/df61dea5fa8b18d1e0044dc0db6702b053d5119f)) +* key the large root font size on screen width, not window width ([#11216](https://github.com/windmill-labs/windmill/issues/11216)) ([3b4e13d](https://github.com/windmill-labs/windmill/commit/3b4e13d1c564c6195e30b55b0671f7533e3ce408)) +* re-encrypt git sync secrets on workspace key rotation ([#11218](https://github.com/windmill-labs/windmill/issues/11218)) ([f0d66a4](https://github.com/windmill-labs/windmill/commit/f0d66a42eba27502767423af0c08b2ca1f16e8df)) +* re-point cloned fork identities that name nobody in the fork ([#11161](https://github.com/windmill-labs/windmill/issues/11161)) ([9690c44](https://github.com/windmill-labs/windmill/commit/9690c4462cf264a5577b87d07d465b5442b4e09d)) +* register the job token with the sensitive log masking system ([#10943](https://github.com/windmill-labs/windmill/issues/10943)) ([53afecd](https://github.com/windmill-labs/windmill/commit/53afecd4588247bc1812d3e68a30db1f3c3b2724)) +* show the New menu's description panel only on hover ([#11199](https://github.com/windmill-labs/windmill/issues/11199)) ([ecd0a6c](https://github.com/windmill-labs/windmill/commit/ecd0a6c77bc3a057b8072dbb0aca731e8bd3d882)) +* stop picker listing a draft twice when drafts share a friendly path ([#11214](https://github.com/windmill-labs/windmill/issues/11214)) ([72507d5](https://github.com/windmill-labs/windmill/commit/72507d52a021de8101794ee8f4a3392be1ba20c7)) + + +### Performance Improvements + +* count completed jobs on the v2_job_completed index with a timeout ([#11211](https://github.com/windmill-labs/windmill/issues/11211)) ([d1a2536](https://github.com/windmill-labs/windmill/commit/d1a25360b070994a31fb033707dbfde1fb553939)) + +## [1.814.0](https://github.com/windmill-labs/windmill/compare/v1.813.0...v1.814.0) (2026-09-17) + + +### Features + +* **ai-chat:** add list_workers and list_data_metrics global tools ([#11143](https://github.com/windmill-labs/windmill/issues/11143)) ([e954d33](https://github.com/windmill-labs/windmill/commit/e954d33613e4ff5027667eb8f646615d9bbd499d)) +* **ai-chat:** merge get_job_logs and get_flow_run_details into get_run ([#11172](https://github.com/windmill-labs/windmill/issues/11172)) ([5bb37ca](https://github.com/windmill-labs/windmill/commit/5bb37ca3388666fba72c55534e37f37bb3e9299e)) +* allow git sync auto-pull, promotion and PRs on Pro licenses ([#11173](https://github.com/windmill-labs/windmill/issues/11173)) ([02e47de](https://github.com/windmill-labs/windmill/commit/02e47de8b4c4f3f54753aabf8c67bc8e71ffb957)) +* badge chat-input flows on the home list ([#11164](https://github.com/windmill-labs/windmill/issues/11164)) ([3d08197](https://github.com/windmill-labs/windmill/commit/3d0819718221f885b61e73d02b43dcc853c7d02a)) +* collect flow conversations and agent memory once their last message goes ([#11178](https://github.com/windmill-labs/windmill/issues/11178)) ([23c24a9](https://github.com/windmill-labs/windmill/commit/23c24a9688d4c8c462f53221334d538280f16bca)) +* flow chat model picker on a shared model-settings component ([#11187](https://github.com/windmill-labs/windmill/issues/11187)) ([189793c](https://github.com/windmill-labs/windmill/commit/189793c2e4db7f1c853695ebcc895c1ec82ed19f)) +* keep flow inputs and seed the agent when chat mode is enabled ([#11177](https://github.com/windmill-labs/windmill/issues/11177)) ([68f2248](https://github.com/windmill-labs/windmill/commit/68f2248018fc218a090bf939e1eb22ff97d5bc22)) +* let plan mode search and read connected mcp servers ([#11205](https://github.com/windmill-labs/windmill/issues/11205)) ([5371519](https://github.com/windmill-labs/windmill/commit/5371519f0f5ce7750982dcdb374dca72115902e7)) +* let test_run_flow name the conversation of a chat-mode test run ([#11198](https://github.com/windmill-labs/windmill/issues/11198)) ([6e1ef93](https://github.com/windmill-labs/windmill/commit/6e1ef93f329cb396ffc3df3304d592e8fa0e0e71)) +* managed memory with an inherited or custom memory id per step ([#11118](https://github.com/windmill-labs/windmill/issues/11118)) ([c297ed0](https://github.com/windmill-labs/windmill/commit/c297ed0052d998fb8f063faa2a36c6eb03e327be)) +* render the flow chat through the shared session chat components ([#11175](https://github.com/windmill-labs/windmill/issues/11175)) ([a9ec0ae](https://github.com/windmill-labs/windmill/commit/a9ec0aec3ac0c6b0f7919d0eb2168816923826d7)) +* show flow step detail inside the graph tab on narrow detail layouts ([#11168](https://github.com/windmill-labs/windmill/issues/11168)) ([64dffe6](https://github.com/windmill-labs/windmill/commit/64dffe6106ad6a55b61a423c855a4b5b0cef533e)) +* store mcp tool call, result and reasoning on flow conversation rows ([#11176](https://github.com/windmill-labs/windmill/issues/11176)) ([a571117](https://github.com/windmill-labs/windmill/commit/a571117f3fd2cef14c920770645c60ee358fdfdd)) +* tell test flow conversations from deployed ones and rename a chat ([#11179](https://github.com/windmill-labs/windmill/issues/11179)) ([4eab995](https://github.com/windmill-labs/windmill/commit/4eab995cf7cf091a5e4640da4cb77e0921bb7fdf)) + + +### Bug Fixes + +* disable a schedule whose cron has no run left instead of panicking ([#11195](https://github.com/windmill-labs/windmill/issues/11195)) ([381d447](https://github.com/windmill-labs/windmill/commit/381d4470ef699ea82283742132e56556b95d2bd2)) +* skip expiry notifications for app embed and SDK tokens ([#11169](https://github.com/windmill-labs/windmill/issues/11169)) ([9d348f8](https://github.com/windmill-labs/windmill/commit/9d348f84c7830f36b6153472556fd70e3d84cd24)) + +## [1.813.0](https://github.com/windmill-labs/windmill/compare/v1.812.0...v1.813.0) (2026-09-16) + + +### Features + +* back AI sessions up to the workspace object storage ([#11116](https://github.com/windmill-labs/windmill/issues/11116)) ([796b6e5](https://github.com/windmill-labs/windmill/commit/796b6e5297d8cceb842ec097f33ec1c3115058bd)) +* delete a browser's copy of an AI session past its workspace retention ([#11156](https://github.com/windmill-labs/windmill/issues/11156)) ([a48ae65](https://github.com/windmill-labs/windmill/commit/a48ae656ae59d600311f81ef357d08df5226515a)) +* rename saved agents from the agent editor and flag broken links ([#11147](https://github.com/windmill-labs/windmill/issues/11147)) ([57a99f6](https://github.com/windmill-labs/windmill/commit/57a99f66a88f195cac8f583b59d69d627cc1ec1d)) +* retention for AI sessions on the object store and in the browser ([#11152](https://github.com/windmill-labs/windmill/issues/11152)) ([ee6d317](https://github.com/windmill-labs/windmill/commit/ee6d317e318fa8a1506fb18d51c45b627070627a)) +* return an ai agent step's thinking in its job result ([#11140](https://github.com/windmill-labs/windmill/issues/11140)) ([c4e878e](https://github.com/windmill-labs/windmill/commit/c4e878e8313a72a16bfbe81fbb3935ee7728ec6f)) +* stream reasoning summaries in AI agent Responses API steps ([#11124](https://github.com/windmill-labs/windmill/issues/11124)) ([b51c0ea](https://github.com/windmill-labs/windmill/commit/b51c0eabbe774c78a9cbf824a3df6528d970b8f6)) + + +### Bug Fixes + +* **apps:** re-check access in place after a password sign-in ([#11166](https://github.com/windmill-labs/windmill/issues/11166)) ([49d0310](https://github.com/windmill-labs/windmill/commit/49d0310ecc08040b6c6fa4f402584543d679e2e8)) +* **apps:** run-mode inline app component uses only pinned content ([#11135](https://github.com/windmill-labs/windmill/issues/11135)) ([781b5a5](https://github.com/windmill-labs/windmill/commit/781b5a57e81eb721d97d7b87e23dd84f23895400)) +* **cli:** keep the workspace color when settings are synced from git ([#11144](https://github.com/windmill-labs/windmill/issues/11144)) ([129c045](https://github.com/windmill-labs/windmill/commit/129c04559548cd1bcf67758ec416fb2a48e7b928)) +* **cli:** resolve lockgen imports through modules a push leaves alone ([#11160](https://github.com/windmill-labs/windmill/issues/11160)) ([54553b2](https://github.com/windmill-labs/windmill/commit/54553b2add6941395f03ca34ee24cf335a3b23c2)) +* dispatch workflow-as-code tasks from a deployed flow's inline step ([#11146](https://github.com/windmill-labs/windmill/issues/11146)) ([e8078f2](https://github.com/windmill-labs/windmill/commit/e8078f2a963166b09849650424583f5dcfd28a84)) +* keep sidebar confirmation dialogs from being confined to the rail ([#11158](https://github.com/windmill-labs/windmill/issues/11158)) ([b9b5988](https://github.com/windmill-labs/windmill/commit/b9b5988ebdf3edd75add445282977c516ef5ed51)) +* keep the instance users table's actions and header in view ([#11145](https://github.com/windmill-labs/windmill/issues/11145)) ([a9a9335](https://github.com/windmill-labs/windmill/commit/a9a9335a34a13ffd8cd2699adc92087b679548ca)) +* stop reading an array job result as wm_failure or http response ([#11154](https://github.com/windmill-labs/windmill/issues/11154)) ([9a8a9c4](https://github.com/windmill-labs/windmill/commit/9a8a9c480cf008761ec6ceaff694085a252a32ae)) +* walk the whole fork ancestry for app installations and fork conflicts ([#11151](https://github.com/windmill-labs/windmill/issues/11151)) ([73dc892](https://github.com/windmill-labs/windmill/commit/73dc892f9c9c840a5f0fb12fcbb28bbe0f38795f)) +* **worker:** bound cache transfers and import fetches in bun jobs ([#11138](https://github.com/windmill-labs/windmill/issues/11138)) ([31c4325](https://github.com/windmill-labs/windmill/commit/31c43255fdcc827c3fdf65e40238f8f3a83201cd)) + +## [1.812.0](https://github.com/windmill-labs/windmill/compare/v1.811.1...v1.812.0) (2026-09-15) + + +### Features + +* add per-route CORS origin allowlist for HTTP triggers ([#10833](https://github.com/windmill-labs/windmill/issues/10833)) ([d8d7332](https://github.com/windmill-labs/windmill/commit/d8d7332eb6d92f7a55b82890de5de4196039b40d)) +* **ai-sessions:** share session artifacts with the workspace by link ([#11115](https://github.com/windmill-labs/windmill/issues/11115)) ([57a134e](https://github.com/windmill-labs/windmill/commit/57a134e2de18e27b3c1d3066a60b892af1089b30)) +* **cli:** list, get and restore trashed items with wmill trash ([#11125](https://github.com/windmill-labs/windmill/issues/11125)) ([a95e950](https://github.com/windmill-labs/windmill/commit/a95e950529f6f01a220e09d322d5a4fb507ea694)) +* dynamic AI agent toolsets ([#11050](https://github.com/windmill-labs/windmill/issues/11050)) ([a78beff](https://github.com/windmill-labs/windmill/commit/a78beff743f6bb289805c263a8d32515bea9688f)) +* **git-sync:** gate GitHub PRs on Windmill CI test results (WIN-2051) ([#10096](https://github.com/windmill-labs/windmill/issues/10096)) ([80eba80](https://github.com/windmill-labs/windmill/commit/80eba80d6ed51753cfaa67310f1a0f5dd5ce0484)) +* pre-approved cloud accounts: login links, OAuth adoption, setup, and the trial bridge ([#10875](https://github.com/windmill-labs/windmill/issues/10875)) ([91e6dc3](https://github.com/windmill-labs/windmill/commit/91e6dc39ce795fafc2bed0d799b62c9880fd6430)) +* run a flow step test through the chat's argument form ([#11114](https://github.com/windmill-labs/windmill/issues/11114)) ([5d32b61](https://github.com/windmill-labs/windmill/commit/5d32b6106788b665e4752fcac63ff1ef457d604e)) +* store resource type display names and label hub integrations ([#11113](https://github.com/windmill-labs/windmill/issues/11113)) ([42f4896](https://github.com/windmill-labs/windmill/commit/42f489685bc87a8479818175c87b5a21b0c17998)) +* windmill-chat sdk for chat-mode flows in external frontends and raw apps ([#11117](https://github.com/windmill-labs/windmill/issues/11117)) ([e8c02c0](https://github.com/windmill-labs/windmill/commit/e8c02c04cdb1a3f199f3f0a8b53a839a53d4a1d9)) + + +### Bug Fixes + +* **ai-chat:** hide other users' MCP servers from the chat unless shared ([#11112](https://github.com/windmill-labs/windmill/issues/11112)) ([244ec13](https://github.com/windmill-labs/windmill/commit/244ec132914a6e689d7d79da9cf2cb39f920aaef)) +* **cli:** say where a sync push deleted variable or resource went ([#10851](https://github.com/windmill-labs/windmill/issues/10851)) ([d54a66f](https://github.com/windmill-labs/windmill/commit/d54a66f15c09c34f7a2b45c2e6e9649807a19265)) +* **cli:** stage a rewritten shared lockfile on git-sync deploy push ([#11126](https://github.com/windmill-labs/windmill/issues/11126)) ([75ee497](https://github.com/windmill-labs/windmill/commit/75ee497011dca076de0923ded9da8e23e08bfb84)) +* **flows:** stop re-evaluating skip_if once a loop is in progress ([#11008](https://github.com/windmill-labs/windmill/issues/11008)) ([56e21bc](https://github.com/windmill-labs/windmill/commit/56e21bce832182528562688acd13ec416e01ebfc)) +* **git-sync:** run auto-pull as the admin who enabled it ([#11121](https://github.com/windmill-labs/windmill/issues/11121)) ([69e6efd](https://github.com/windmill-labs/windmill/commit/69e6efd875e020779ea115c096331da8eae2ffe7)) +* keep a script draft's password marking through the chat's run form ([#11110](https://github.com/windmill-labs/windmill/issues/11110)) ([56dd940](https://github.com/windmill-labs/windmill/commit/56dd940e34b146c3ca25de7959b4fb618308eb86)) +* **python:** parse wheel RECORD paths as RFC 4180 csv fields ([#11133](https://github.com/windmill-labs/windmill/issues/11133)) ([9696308](https://github.com/windmill-labs/windmill/commit/96963080f1711192fe1d9bc4142c1710511504d4)) +* re-attach flow chat to the same job on SSE timeout instead of re-running it ([#11122](https://github.com/windmill-labs/windmill/issues/11122)) ([94c548c](https://github.com/windmill-labs/windmill/commit/94c548cd5dc43131e0471b0edabe496dff245862)) +* set the enclosing span's trace context on exported log records ([#11123](https://github.com/windmill-labs/windmill/issues/11123)) ([d0cac08](https://github.com/windmill-labs/windmill/commit/d0cac0807f1b6f4fc76d6e5f7e27e03d30abec8d)) +* skip instance group members that are not email addresses ([#11128](https://github.com/windmill-labs/windmill/issues/11128)) ([e3e638f](https://github.com/windmill-labs/windmill/commit/e3e638f7f587f0ee090d06436575b65a0860a855)) +* wake a WAC parent from every path that completes its child ([#11119](https://github.com/windmill-labs/windmill/issues/11119)) ([0b1e9c0](https://github.com/windmill-labs/windmill/commit/0b1e9c0dda2ae55c56b1e0de5c0419b4511b973f)) + +## [1.811.1](https://github.com/windmill-labs/windmill/compare/v1.811.0...v1.811.1) (2026-09-13) + + +### Bug Fixes + +* check kafka trigger topics against a set, not a one-pass iterator ([#11108](https://github.com/windmill-labs/windmill/issues/11108)) ([bf4fa2b](https://github.com/windmill-labs/windmill/commit/bf4fa2b174b8d4a5897b2aa0d108480442bd0e18)) +* let the hub_sync job read the uid and hub_base_url settings ([#11106](https://github.com/windmill-labs/windmill/issues/11106)) ([45102c8](https://github.com/windmill-labs/windmill/commit/45102c82659d86ca5ec6fd3aee57232f2348736c)) + +## [1.811.0](https://github.com/windmill-labs/windmill/compare/v1.810.0...v1.811.0) (2026-09-12) + + +### Features + +* make snowflake_oauth work as a dbt warehouse on every engine ([#11095](https://github.com/windmill-labs/windmill/issues/11095)) ([9fc50a2](https://github.com/windmill-labs/windmill/commit/9fc50a23fb75cb541c481247b7b25c206fb06d36)) + + +### Bug Fixes + +* accept any hub version of the git sync script in the token check ([#11099](https://github.com/windmill-labs/windmill/issues/11099)) ([670628b](https://github.com/windmill-labs/windmill/commit/670628b300ab119363adb5496ebfe3c6ccd80063)) +* bring back Publish to Hub for scripts ([#11097](https://github.com/windmill-labs/windmill/issues/11097)) ([864e5f0](https://github.com/windmill-labs/windmill/commit/864e5f02ec1c2dd16c74524f551f44485df10a20)) +* bundle deployed bun scripts whose only pin is on a dynamic import ([#11096](https://github.com/windmill-labs/windmill/issues/11096)) ([4afb9aa](https://github.com/windmill-labs/windmill/commit/4afb9aa677ac11d22e22e54bb6fd7881378b7005)) +* clear a stale git auto-pull failure and show the status time ([#11100](https://github.com/windmill-labs/windmill/issues/11100)) ([e877b5f](https://github.com/windmill-labs/windmill/commit/e877b5f2e81b1741aee90e843c8cccc22f9eef24)) +* stop a resource delete from taking variables it does not own ([#11102](https://github.com/windmill-labs/windmill/issues/11102)) ([2a21efa](https://github.com/windmill-labs/windmill/commit/2a21efa11b8307b331a8c20720028444dd3c62ff)) + +## [1.810.0](https://github.com/windmill-labs/windmill/compare/v1.809.0...v1.810.0) (2026-09-11) + + +### Features + +* **ai-sessions:** turn skills on by default, and group them by folder ([#11058](https://github.com/windmill-labs/windmill/issues/11058)) ([d8b9174](https://github.com/windmill-labs/windmill/commit/d8b9174235b97d0b4ef9f281e20e5704182d8dcb)) +* background and wait_seconds for run_script, skip preprocessor ([#11092](https://github.com/windmill-labs/windmill/issues/11092)) ([2939c2d](https://github.com/windmill-labs/windmill/commit/2939c2dd4b129640d87ef45c7a24c6f215a3239a)) +* give the chat the full MCP tool schema, and mark calls with the provider icon ([#11086](https://github.com/windmill-labs/windmill/issues/11086)) ([e7c6f85](https://github.com/windmill-labs/windmill/commit/e7c6f85553bd8efb0f7af0f488f615ac93c496ae)) +* let apps hide the viewer login status on public urls ([#11089](https://github.com/windmill-labs/windmill/issues/11089)) ([6056ec7](https://github.com/windmill-labs/windmill/commit/6056ec7148bce9f8ed171dd29f544696c335de7d)) +* remove the viewer login status badge from public apps ([#11090](https://github.com/windmill-labs/windmill/issues/11090)) ([75d7bee](https://github.com/windmill-labs/windmill/commit/75d7bee178886461fe49090d606a708f25c02d1a)) +* run a deployed flow through the chat's argument form ([#11085](https://github.com/windmill-labs/windmill/issues/11085)) ([b50de89](https://github.com/windmill-labs/windmill/commit/b50de8947908f1a5a4e9472afe6c0ecd25892e93)) +* run a flow test through the chat's argument form ([#11069](https://github.com/windmill-labs/windmill/issues/11069)) ([172d6c2](https://github.com/windmill-labs/windmill/commit/172d6c275b92b39d13848b543df9db10148e94ef)) + + +### Bug Fixes + +* attach TLS to gRPC OTLP exporters for https endpoints ([#11078](https://github.com/windmill-labs/windmill/issues/11078)) ([f915ed6](https://github.com/windmill-labs/windmill/commit/f915ed6a46e14341ddb213e869090edb68763032)) +* **dbt:** stop dbt sending anonymous usage stats from workers ([#11091](https://github.com/windmill-labs/windmill/issues/11091)) ([d539e86](https://github.com/windmill-labs/windmill/commit/d539e8674f2bf92d6c10b182ed4a1073714062c0)) +* keep pinned import versions of imported scripts in bun lockfiles ([#11082](https://github.com/windmill-labs/windmill/issues/11082)) ([57f8b08](https://github.com/windmill-labs/windmill/commit/57f8b0826ad61cb118d0cafbbc9203327d858940)) +* let admins and background sync reach private git hosts ([#11084](https://github.com/windmill-labs/windmill/issues/11084)) ([fa53099](https://github.com/windmill-labs/windmill/commit/fa53099e2b87a8676c5d6a18e77844e17ff45efd)) +* serve instance env settings at the documented /settings/local path ([#11075](https://github.com/windmill-labs/windmill/issues/11075)) ([f8f7c00](https://github.com/windmill-labs/windmill/commit/f8f7c0009f32c1440566420725b12e78ca804b03)) +* show symlinked files in the git repo viewer ([#11081](https://github.com/windmill-labs/windmill/issues/11081)) ([e6d4f44](https://github.com/windmill-labs/windmill/commit/e6d4f44a6122dab47fbee2a2a2b4330a62841bed)) +* support gzip and zstd compression for OTLP export over gRPC ([#11077](https://github.com/windmill-labs/windmill/issues/11077)) ([b156778](https://github.com/windmill-labs/windmill/commit/b156778da24e3827f723e503f80df68de87ddf7e)) +* unpin only the specifiers in the bundle a bun modules run executes ([#11083](https://github.com/windmill-labs/windmill/issues/11083)) ([30ffdbe](https://github.com/windmill-labs/windmill/commit/30ffdbecc15270562ceed3030c50a1cf81b1195c)) + + +### Performance Improvements + +* lazy-load the low-code runtime on public app pages ([#11087](https://github.com/windmill-labs/windmill/issues/11087)) ([e651b4c](https://github.com/windmill-labs/windmill/commit/e651b4cd63c3bd64a8739c9f60c6dfa437f19b4b)) + +## [1.809.0](https://github.com/windmill-labs/windmill/compare/v1.808.0...v1.809.0) (2026-09-10) + + +### Features + +* add a minimal skin for the approval page and slack/teams ([#11061](https://github.com/windmill-labs/windmill/issues/11061)) ([63cb46d](https://github.com/windmill-labs/windmill/commit/63cb46d7bb9db1d996aa38e06bd3afebc60111bd)) +* live queue status per tag and bounded queue metric charts ([#11067](https://github.com/windmill-labs/windmill/issues/11067)) ([569adb8](https://github.com/windmill-labs/windmill/commit/569adb85c1885d289e80a70a166f0f74e6d5ba83)) +* **otel:** read the OTLP metrics temporality preference ([#11064](https://github.com/windmill-labs/windmill/issues/11064)) ([2f88769](https://github.com/windmill-labs/windmill/commit/2f8876908719b3640d7cfc9364a7b0f7145fc356)) +* **otel:** support standard OTEL resource attribute env vars ([#10974](https://github.com/windmill-labs/windmill/issues/10974)) ([0a40eea](https://github.com/windmill-labs/windmill/commit/0a40eea37a7dbde5fc4760d6333d81186dfee255)) +* report script metadata with no content file in wmill lint ([#11053](https://github.com/windmill-labs/windmill/issues/11053)) ([8820b9f](https://github.com/windmill-labs/windmill/commit/8820b9fc644c6620c50517cd0d902015e2e670e2)) +* show the workspace an operator is in, and let them switch ([#11059](https://github.com/windmill-labs/windmill/issues/11059)) ([385086f](https://github.com/windmill-labs/windmill/commit/385086ffc21c72cd07624584932a4a301c23a732)) +* tuck other users' spaces into a collapsible home tree row ([#11073](https://github.com/windmill-labs/windmill/issues/11073)) ([d87f089](https://github.com/windmill-labs/windmill/commit/d87f089288996af9ea7e3b017a5ef35d4ded880d)) + + +### Bug Fixes + +* **ai-chat:** test_run_flow could test a different flow than the one asked ([#11066](https://github.com/windmill-labs/windmill/issues/11066)) ([fa73539](https://github.com/windmill-labs/windmill/commit/fa73539839071491fb2cbad9242f52ad22b975bf)) +* bound list_jobs runtime and paginate runs on the sorted column ([#11072](https://github.com/windmill-labs/windmill/issues/11072)) ([f517402](https://github.com/windmill-labs/windmill/commit/f51740253871960b55ab2aa8989e8df3fbde0351)) +* **frontend:** clear the flow graph selection through xyflow's store ([#11056](https://github.com/windmill-labs/windmill/issues/11056)) ([b4be8bc](https://github.com/windmill-labs/windmill/commit/b4be8bc5354fbd3a47c267c331aaf603c0f90e6e)) +* **frontend:** recompute dataflow edges when selecting a step ([#11070](https://github.com/windmill-labs/windmill/issues/11070)) ([08d876a](https://github.com/windmill-labs/windmill/commit/08d876aebf32ebb995a8c1839aa87794c0176bed)) +* **frontend:** restore heading sizes in note markdown and keep group notes on id change ([#11047](https://github.com/windmill-labs/windmill/issues/11047)) ([e63072c](https://github.com/windmill-labs/windmill/commit/e63072c216383700a23504be89782f4c69657174)) +* give every table a primary key so the db can be logically replicated ([#11036](https://github.com/windmill-labs/windmill/issues/11036)) ([e62bfdc](https://github.com/windmill-labs/windmill/commit/e62bfdcd8c6f1389601ccd2b5809c5eff0ff262e)) +* keep an app's deployed policy on wmill push ([#11049](https://github.com/windmill-labs/windmill/issues/11049)) ([0af7675](https://github.com/windmill-labs/windmill/commit/0af7675588300863883a99c7f14bc53cafec8a7e)) +* refuse cross-site GET requests that run Hub scripts ([#11054](https://github.com/windmill-labs/windmill/issues/11054)) ([ab9efc8](https://github.com/windmill-labs/windmill/commit/ab9efc897cc94d61a67263772806b15e9225cece)) +* skip the deploy PR when the git sync push committed nothing ([#11076](https://github.com/windmill-labs/windmill/issues/11076)) ([8ecbd33](https://github.com/windmill-labs/windmill/commit/8ecbd339eef7314a93d599fcea4377299d4c493d)) +* space the trailing AI settings cards ([#11044](https://github.com/windmill-labs/windmill/issues/11044)) ([5d7eed1](https://github.com/windmill-labs/windmill/commit/5d7eed1c02b0966289cc8cca00a15d76dad187a5)) +* stop uv pip compile emitting lockfile annotations ([#11042](https://github.com/windmill-labs/windmill/issues/11042)) ([8aa8b7e](https://github.com/windmill-labs/windmill/commit/8aa8b7ee6c23f859f169637c0bfd3f8d964509e9)) +* surface why a private or untrusted git host is unreachable ([#11068](https://github.com/windmill-labs/windmill/issues/11068)) ([c57b18e](https://github.com/windmill-labs/windmill/commit/c57b18e46fcdd319213fe0a537cb18de418ca688)) + + +### Performance Improvements + +* index the FK columns that cascade on workspace delete ([#11052](https://github.com/windmill-labs/windmill/issues/11052)) ([9a563f6](https://github.com/windmill-labs/windmill/commit/9a563f6d72da28fe09b785cd0683e9698df72bba)) +* only write queue metrics when a tag's backlog changes ([#11055](https://github.com/windmill-labs/windmill/issues/11055)) ([9d75929](https://github.com/windmill-labs/windmill/commit/9d75929247ea2ec39286971fcbebf95d886194f3)) + +## [1.808.0](https://github.com/windmill-labs/windmill/compare/v1.807.0...v1.808.0) (2026-09-09) + + +### Features + +* run and test scripts from the AI chat through an argument form ([#11001](https://github.com/windmill-labs/windmill/issues/11001)) ([a6abf2c](https://github.com/windmill-labs/windmill/commit/a6abf2c8a744e9ee6acf4830cdfbb84f2f95cb36)) + +## [1.807.0](https://github.com/windmill-labs/windmill/compare/v1.806.0...v1.807.0) (2026-09-09) + + +### Features + +* add a dismissible instance-wide announcement banner ([#11037](https://github.com/windmill-labs/windmill/issues/11037)) ([abf4c6c](https://github.com/windmill-labs/windmill/commit/abf4c6c2348014ea4401b9be62b5b15e5800e879)) +* batch chained DDL statements into a single migration ([#11038](https://github.com/windmill-labs/windmill/issues/11038)) ([656e609](https://github.com/windmill-labs/windmill/commit/656e609595833bc854f845d35cf43157e76d732f)) +* create the cloud workspace in onboarding, and teach the empty home ([#10959](https://github.com/windmill-labs/windmill/issues/10959)) ([fd35b47](https://github.com/windmill-labs/windmill/commit/fd35b4765843879cb2254f402c142fd7510f1916)) +* link from the public run view to the authenticated run page ([#11041](https://github.com/windmill-labs/windmill/issues/11041)) ([09b81a9](https://github.com/windmill-labs/windmill/commit/09b81a9294bed2795a1a7b688d0b02e0a61957ea)) +* make guest access unavailable on the shared cloud ([#11040](https://github.com/windmill-labs/windmill/issues/11040)) ([0b63e0a](https://github.com/windmill-labs/windmill/commit/0b63e0a6929088ff25def4fd6547cf61668251a5)) + + +### Bug Fixes + +* ignore comments and continuations in python lockfiles ([#11035](https://github.com/windmill-labs/windmill/issues/11035)) ([90c4e10](https://github.com/windmill-labs/windmill/commit/90c4e1020a2ff896977648dd68b572413cea7709)) +* refetch an unparseable hub script cache entry instead of panicking ([#11033](https://github.com/windmill-labs/windmill/issues/11033)) ([88c3ebd](https://github.com/windmill-labs/windmill/commit/88c3ebdfc1325ffbea4521d854e71d231a6409c4)) +* stop a new AI session adopting a legacy sidebar chat ([#11039](https://github.com/windmill-labs/windmill/issues/11039)) ([1076b63](https://github.com/windmill-labs/windmill/commit/1076b638d987ba99c5c27e5478ca580534b9d572)) + +## [1.806.0](https://github.com/windmill-labs/windmill/compare/v1.805.0...v1.806.0) (2026-09-08) + + +### Features + +* bring gitlab repositories to parity for git sync ([#10938](https://github.com/windmill-labs/windmill/issues/10938)) ([9444049](https://github.com/windmill-labs/windmill/commit/9444049d6013c77a5f25f01a736eb5cd741fb3e6)) +* draw a dbt column trace, across projects and the pipeline boundary ([#11014](https://github.com/windmill-labs/windmill/issues/11014)) ([33f9828](https://github.com/windmill-labs/windmill/commit/33f9828c3ed15fe63fccedc1550584f15c0490ab)) +* durable dbt state per environment, and `--defer` onto it ([#10975](https://github.com/windmill-labs/windmill/issues/10975)) ([621fac5](https://github.com/windmill-labs/windmill/commit/621fac55abcd1859e8c8c06e5f4412e61bb85d59)) +* ingest dbt column lineage and real column schemas from the engine's parquet index ([#10977](https://github.com/windmill-labs/windmill/issues/10977)) ([0139467](https://github.com/windmill-labs/windmill/commit/0139467b01b82e4b3d474ca3f205358fa607d19a)) +* let a worker group override the dependency cache object store ([#11019](https://github.com/windmill-labs/windmill/issues/11019)) ([de98adf](https://github.com/windmill-labs/windmill/commit/de98adf055835ab7c4d6305e7d5d3bdab915876b)) +* **nativets:** bound fetch on a peer that never answers ([#11026](https://github.com/windmill-labs/windmill/issues/11026)) ([785277e](https://github.com/windmill-labs/windmill/commit/785277e0bb2ea77b71a89dd4d389d439dfcf9e03)) +* recognize `// volume:` mounts in PHP scripts ([#11018](https://github.com/windmill-labs/windmill/issues/11018)) ([f081fb1](https://github.com/windmill-labs/windmill/commit/f081fb10705cadf99e99dfa786d1cc2ebf0447db)) +* report a WAC task failure the workflow body never awaited ([#11017](https://github.com/windmill-labs/windmill/issues/11017)) ([3e3a41d](https://github.com/windmill-labs/windmill/commit/3e3a41d418d4ee3fe060bd3acf3324f311d89c8a)) +* retry a workflow-as-code task from its task options ([#11013](https://github.com/windmill-labs/windmill/issues/11013)) ([d3f305d](https://github.com/windmill-labs/windmill/commit/d3f305db982b7c5dc49babf9bec8b62adcd2557d)) + + +### Bug Fixes + +* chain redeploys onto a retired path's version history ([#11029](https://github.com/windmill-labs/windmill/issues/11029)) ([0b37226](https://github.com/windmill-labs/windmill/commit/0b372260787edb7e9627ad4fb6637ad5f1024e0a)) +* make the native trigger disable/enable toggle actually save ([#11024](https://github.com/windmill-labs/windmill/issues/11024)) ([448fce9](https://github.com/windmill-labs/windmill/commit/448fce93f743d5b2ef2a2d4496eb2ec238594a0a)) +* offload php signature parsing from async workers ([#11027](https://github.com/windmill-labs/windmill/issues/11027)) ([2cb02e3](https://github.com/windmill-labs/windmill/commit/2cb02e3b3398db49f16377dd79dde2dd6fb5cc02)) +* reduce php parser stack use in debug workers ([#11025](https://github.com/windmill-labs/windmill/issues/11025)) ([2ae8509](https://github.com/windmill-labs/windmill/commit/2ae8509b14f112c9ef5b71321e8fd52596a16c10)) + + +### Performance Improvements + +* reduce shared worker debug polling frames ([#11028](https://github.com/windmill-labs/windmill/issues/11028)) ([946756a](https://github.com/windmill-labs/windmill/commit/946756ae83deb4e7a93111edddbd4d98c596d5a9)) + +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + +## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) + + +### Features + +* **ai-sessions:** replace the context panel with an assistant settings modal ([#10919](https://github.com/windmill-labs/windmill/issues/10919)) ([fda7b3f](https://github.com/windmill-labs/windmill/commit/fda7b3f086619e3716e5894c07be127104174f1d)) +* **frontend:** group the agent form and edit saved agents as drafts ([#10880](https://github.com/windmill-labs/windmill/issues/10880)) ([f037c73](https://github.com/windmill-labs/windmill/commit/f037c73d104fffe7bb2640a5b1f2a92154c85e06)) +* guest app execution mode, a role that takes no seat ([#10929](https://github.com/windmill-labs/windmill/issues/10929)) ([fce635d](https://github.com/windmill-labs/windmill/commit/fce635d3c4c8962f448140ceb55a00fb99012701)) +* guest JWT entry for embedded apps ([#10954](https://github.com/windmill-labs/windmill/issues/10954)) ([8aab503](https://github.com/windmill-labs/windmill/commit/8aab5034a68a4aafb264b0e86d000ef58f4a8511)) +* instrument sandbox isolation, data tables and in-flow script edits ([#10981](https://github.com/windmill-labs/windmill/issues/10981)) ([130a2f7](https://github.com/windmill-labs/windmill/commit/130a2f74083ba1bd308beeb86e2cbbaa41fd3345)) +* make S3 permission rules reorderable by drag and drop ([#10958](https://github.com/windmill-labs/windmill/issues/10958)) ([2257b05](https://github.com/windmill-labs/windmill/commit/2257b05b2857c7ae2b5ae0b4f9004e2d4e757925)) +* reconcile IdP instance groups from the SSO groups claim ([#10957](https://github.com/windmill-labs/windmill/issues/10957)) ([79426a1](https://github.com/windmill-labs/windmill/commit/79426a1a68a6b19e12af4633b8a79d07a103a106)) + + +### Bug Fixes + +* deploy a relocked script version only when its lock changed ([#10966](https://github.com/windmill-labs/windmill/issues/10966)) ([1113828](https://github.com/windmill-labs/windmill/commit/11138284acc4c1d8673e86823c7f74c9e1f419e6)) +* **frontend:** render ordered lists in markdown descriptions ([#10973](https://github.com/windmill-labs/windmill/issues/10973)) ([a0295b2](https://github.com/windmill-labs/windmill/commit/a0295b20c436fd3f2bd6a6d294ae3cee005391e8)) +* keep braces inside string tool arguments out of JSON depth count ([#10965](https://github.com/windmill-labs/windmill/issues/10965)) ([3e3d2a6](https://github.com/windmill-labs/windmill/commit/3e3d2a636334146014926841949372083e6e8516)) +* keep the instance user editor popover inside the viewport ([#10979](https://github.com/windmill-labs/windmill/issues/10979)) ([1901d31](https://github.com/windmill-labs/windmill/commit/1901d3193bfc6a9e29d0b7c5389fef44ff9d3687)) +* meter WAC compute per segment, not the whole sleep ([#10985](https://github.com/windmill-labs/windmill/issues/10985)) ([5428710](https://github.com/windmill-labs/windmill/commit/54287102b22dd17903cdd4b48c5828875e5b9be4)) +* name the extension to load when duckdb autoload hits the fence ([#10972](https://github.com/windmill-labs/windmill/issues/10972)) ([64b6798](https://github.com/windmill-labs/windmill/commit/64b679879936e2ddf4dbc2f90edbd56e3893bd83)) +* **oauth:** show the account chooser on an explicit Google/Microsoft login ([#10961](https://github.com/windmill-labs/windmill/issues/10961)) ([9f7908e](https://github.com/windmill-labs/windmill/commit/9f7908e2622647388768b574083cc48a6e1990f1)) +* patch critical CVEs in the worker image ([#10962](https://github.com/windmill-labs/windmill/issues/10962)) ([b100606](https://github.com/windmill-labs/windmill/commit/b100606da6a61f2dbcb24516363f43643bc917e3)) +* render the MCP OAuth consent page without a workspace ([#10988](https://github.com/windmill-labs/windmill/issues/10988)) ([ebfac29](https://github.com/windmill-labs/windmill/commit/ebfac29096f12c4da2df45d5d82db83d352f3426)) +* stand the WAC park down for a cancel that beat it to the row ([#10990](https://github.com/windmill-labs/windmill/issues/10990)) ([f977f5b](https://github.com/windmill-labs/windmill/commit/f977f5bf8b1ac70d3afbdc8ad6fcbe072cc51ebc)) + +## [1.803.0](https://github.com/windmill-labs/windmill/compare/v1.802.0...v1.803.0) (2026-09-03) + + +### Features + +* expose request headers to scripts invoked via MCP ([#10903](https://github.com/windmill-labs/windmill/issues/10903)) ([e474e88](https://github.com/windmill-labs/windmill/commit/e474e8803ce2ff5c2df09a58dab51d45f5c922ca)) +* reuse an existing workspace resource in the project import wizard ([#10935](https://github.com/windmill-labs/windmill/issues/10935)) ([582761e](https://github.com/windmill-labs/windmill/commit/582761e37c776e92dc1c6ebfee8c4efe7c35d822)) + + +### Bug Fixes + +* bump git sync hub scripts to cli 1.802.1, test the fork ui pull ([#10955](https://github.com/windmill-labs/windmill/issues/10955)) ([ca88009](https://github.com/windmill-labs/windmill/commit/ca8800959aa6a0017cc29bad187c9f49e0d13cc4)) +* **cli:** make a sync push into a fork converge on schedules and inline names ([#10951](https://github.com/windmill-labs/windmill/issues/10951)) ([0f5a1db](https://github.com/windmill-labs/windmill/commit/0f5a1db2abba8df30a2f975f4498e269f13cf93d)) +* fade the home Build with AI placeholder every 10s instead of typing it ([#10953](https://github.com/windmill-labs/windmill/issues/10953)) ([3d089b5](https://github.com/windmill-labs/windmill/commit/3d089b57344f5814086e6176301c5031dc519674)) +* let operators use wmill.datatable() from within running jobs ([#10931](https://github.com/windmill-labs/windmill/issues/10931)) ([9b64a89](https://github.com/windmill-labs/windmill/commit/9b64a89cd46ae718d6c58fa12f925fa041fb1032)) + +## [1.802.0](https://github.com/windmill-labs/windmill/compare/v1.801.0...v1.802.0) (2026-09-02) + + +### Features + +* add retention cleanup for the otel_traces table ([#10949](https://github.com/windmill-labs/windmill/issues/10949)) ([d472193](https://github.com/windmill-labs/windmill/commit/d472193e5bf5f6428e0096a402eb2c9299634fb2)) +* open path links from chat messages in the session preview panel ([#10881](https://github.com/windmill-labs/windmill/issues/10881)) ([f10ac6c](https://github.com/windmill-labs/windmill/commit/f10ac6c2b3644fb16697e650efbc4f7cd3c6944c)) +* restore owner and label filter chips on the homepage ([#10942](https://github.com/windmill-labs/windmill/issues/10942)) ([ccf8476](https://github.com/windmill-labs/windmill/commit/ccf84761dd664b9228dfe2f65867e8c32cd20c21)) +* **sessions:** offer the item you came from when starting a new session ([#10940](https://github.com/windmill-labs/windmill/issues/10940)) ([d3747d6](https://github.com/windmill-labs/windmill/commit/d3747d62555ebcb09c78cfabcaa3b6177758d6ea)) +* workspace setting to hide the AI assistant, agent steps unaffected ([#10941](https://github.com/windmill-labs/windmill/issues/10941)) ([fdd3b36](https://github.com/windmill-labs/windmill/commit/fdd3b36423344a2e1a464674179406581074e926)) + + +### Bug Fixes + +* apply object-storage test SSRF validation to all non-super-admins ([#10933](https://github.com/windmill-labs/windmill/issues/10933)) ([4fef119](https://github.com/windmill-labs/windmill/commit/4fef1195adaa9fa036a219884bd6c996460ca37f)) +* connect to dev server instead of localhost ([#10912](https://github.com/windmill-labs/windmill/issues/10912)) ([337154b](https://github.com/windmill-labs/windmill/commit/337154b8304a5969f35216add627b5c1153c0f6c)) +* preselect first row of AI agent and AI sandbox insert panes ([#10937](https://github.com/windmill-labs/windmill/issues/10937)) ([95b6bbd](https://github.com/windmill-labs/windmill/commit/95b6bbd46ada11d96a914ae5b0e92aba4dd02530)) +* record supplied script lock hashes so importers can skip relocking ([#10915](https://github.com/windmill-labs/windmill/issues/10915)) ([17ba521](https://github.com/windmill-labs/windmill/commit/17ba521c352aec65a8270893752bbadd7f3d6eaa)) +* sandbox script-controlled content types in result_to_response ([#10932](https://github.com/windmill-labs/windmill/issues/10932)) ([419741e](https://github.com/windmill-labs/windmill/commit/419741e5d226c67c51429094fb6ded9474afed99)) + +## [1.801.0](https://github.com/windmill-labs/windmill/compare/v1.800.1...v1.801.0) (2026-09-01) + + +### Features + +* **ai-chat:** make reusable skills ai_skill resources you select per workspace ([#10914](https://github.com/windmill-labs/windmill/issues/10914)) ([cfcfe29](https://github.com/windmill-labs/windmill/commit/cfcfe298dd9ab50196bd64926ef78c4563f58c2c)) +* **ai-sessions:** show a running session across tabs and reload finished turns ([#10916](https://github.com/windmill-labs/windmill/issues/10916)) ([816dc9d](https://github.com/windmill-labs/windmill/commit/816dc9dcd2c310e499d2d210a0abcd403469f29c)) +* edit folders and groups in a drawer that saves once ([#10873](https://github.com/windmill-labs/windmill/issues/10873)) ([5d5ad4e](https://github.com/windmill-labs/windmill/commit/5d5ad4e8974e076ef53a26a5584e4209255a2248)) +* make the home Build with AI composer dismissible, quiet the rest of the home page ([#10930](https://github.com/windmill-labs/windmill/issues/10930)) ([772fafe](https://github.com/windmill-labs/windmill/commit/772fafec8316a1e0c0e76b9a0737cc41d40a9a8c)) + + +### Bug Fixes + +* let a principal without a login account own a draft ([#10925](https://github.com/windmill-labs/windmill/issues/10925)) ([94af8d0](https://github.com/windmill-labs/windmill/commit/94af8d0fb5aceebe83936fd6761c6c1c02c75323)) +* resolve chat path links against the session's operating workspace ([#10924](https://github.com/windmill-labs/windmill/issues/10924)) ([9074de2](https://github.com/windmill-labs/windmill/commit/9074de25ea730ca02653c9a2e2b8b99eda6f3137)) +* tolerate string app_id in GHES app config deserialization ([#10923](https://github.com/windmill-labs/windmill/issues/10923)) ([af8ff38](https://github.com/windmill-labs/windmill/commit/af8ff3868748412cb658c803ebc8a71edc3cd8fb)) + +## [1.800.1](https://github.com/windmill-labs/windmill/compare/v1.800.0...v1.800.1) (2026-09-01) + + +### Bug Fixes + +* add top margin to the home Build with AI section ([#10909](https://github.com/windmill-labs/windmill/issues/10909)) ([bedf5ae](https://github.com/windmill-labs/windmill/commit/bedf5ae57445025729e94e16f1b5f13f6ff38ffa)) +* **ai-chat:** consume an @ mention with the message that carried it ([#10907](https://github.com/windmill-labs/windmill/issues/10907)) ([c512110](https://github.com/windmill-labs/windmill/commit/c512110a1f8d0d3437c20048f6446ef62b10222c)) +* keep a local dbt descriptor under sync pull --keep-deleted ([#10911](https://github.com/windmill-labs/windmill/issues/10911)) ([4b5be38](https://github.com/windmill-labs/windmill/commit/4b5be386ce0f851a087f43c0b0ac6e4b1b055a47)) +* keep windmill-indexer out of builds without tantivy ([#10908](https://github.com/windmill-labs/windmill/issues/10908)) ([db0f004](https://github.com/windmill-labs/windmill/commit/db0f004613e3f90428fea4c824cc53f1b2fc03b0)) + +## [1.800.0](https://github.com/windmill-labs/windmill/compare/v1.799.0...v1.800.0) (2026-08-31) + + +### Features + +* add --keep-deleted flag to wmill sync pull and push ([#10878](https://github.com/windmill-labs/windmill/issues/10878)) ([66123f3](https://github.com/windmill-labs/windmill/commit/66123f3a9b8978c0084b02f50b44cffba125a13a)) +* day-partition the service log index and expire whole chunks ([#10893](https://github.com/windmill-labs/windmill/issues/10893)) ([d91ee46](https://github.com/windmill-labs/windmill/commit/d91ee4614a70f20a194f47e190327129f499ec63)) +* free AI tokens + home search/filter revamp ([#10020](https://github.com/windmill-labs/windmill/issues/10020)) ([716ce2e](https://github.com/windmill-labs/windmill/commit/716ce2ece00cd5cfb8641afc6432636bc4aa46e9)) +* make the service log retention period an instance setting ([#10889](https://github.com/windmill-labs/windmill/issues/10889)) ([815de49](https://github.com/windmill-labs/windmill/commit/815de49e2322f85ca92b1e41a2bcd22591ebe93f)) +* rework the evals dataset drawer and run navigation ([#10884](https://github.com/windmill-labs/windmill/issues/10884)) ([1462f17](https://github.com/windmill-labs/windmill/commit/1462f17643302127b4bd76bacfde80cc03f9d606)) +* serve service log context from parquet and retire the raw log files ([#10892](https://github.com/windmill-labs/windmill/issues/10892)) ([338d75c](https://github.com/windmill-labs/windmill/commit/338d75cc5227e352cb84828c99bfd3b984cf0fa5)) +* serve service log retrieval from a columnar parquet store ([#10886](https://github.com/windmill-labs/windmill/issues/10886)) ([7c1a785](https://github.com/windmill-labs/windmill/commit/7c1a785f756ed27e4425f6534709b19971a73a97)) + + +### Bug Fixes + +* correct the service log ingest flush boundary ([#10898](https://github.com/windmill-labs/windmill/issues/10898)) ([ac56586](https://github.com/windmill-labs/windmill/commit/ac56586c0e56d4022761d3c80306a03d57f8bfcb)) +* harden the service log indexer's recovery and read paths ([#10904](https://github.com/windmill-labs/windmill/issues/10904)) ([831370c](https://github.com/windmill-labs/windmill/commit/831370cdde8e06f4298b17baa1a0041bacdd98c7)) +* keep raw-app editor selection consistent across sidebar and tabs ([#10885](https://github.com/windmill-labs/windmill/issues/10885)) ([b57e231](https://github.com/windmill-labs/windmill/commit/b57e231c2bf5e5fe007f0aa7b958a51e32b47141)) +* register every rotated service log file exactly once ([#10891](https://github.com/windmill-labs/windmill/issues/10891)) ([c817248](https://github.com/windmill-labs/windmill/commit/c8172480b0b1be6c57210212afc71d6ec8711235)) +* show a loading indicator while the initial data table migration is generated ([#10900](https://github.com/windmill-labs/windmill/issues/10900)) ([b998267](https://github.com/windmill-labs/windmill/commit/b998267c91b9dcf02787768f6205cc5aeda494fb)) +* track outstanding service log files on the rows themselves ([#10894](https://github.com/windmill-labs/windmill/issues/10894)) ([aa4a6ff](https://github.com/windmill-labs/windmill/commit/aa4a6ffd66813010a79c07741b01a984ed4e7df6)) + + +### Performance Improvements + +* add service log documents to the index one batch at a time ([#10906](https://github.com/windmill-labs/windmill/issues/10906)) ([0c2eb0a](https://github.com/windmill-labs/windmill/commit/0c2eb0ae3d18f49c21370131d15011e8dd103746)) + +## [1.799.0](https://github.com/windmill-labs/windmill/compare/v1.798.1...v1.799.0) (2026-08-28) + + +### Features + +* enable Anthropic prompt caching on Vertex AI agent steps ([#10876](https://github.com/windmill-labs/windmill/issues/10876)) ([320f400](https://github.com/windmill-labs/windmill/commit/320f4005124202852e6e9c70b394e7f87231d278)) +* instrument AI fill/fix, evals, agents and the debugger ([#10853](https://github.com/windmill-labs/windmill/issues/10853)) ([0bbd559](https://github.com/windmill-labs/windmill/commit/0bbd559ac8a35dba04ba5e8d6f2fd8d1d1124891)) + + +### Bug Fixes + +* **datatables:** stop a fork's pg_dump restore from failing silently ([#10830](https://github.com/windmill-labs/windmill/issues/10830)) ([3ce9bbc](https://github.com/windmill-labs/windmill/commit/3ce9bbc7168b837cb2111aabd533bb67803502b8)) +* key build artifact caches on a runnable's inline modules ([#10819](https://github.com/windmill-labs/windmill/issues/10819)) ([b72ccc3](https://github.com/windmill-labs/windmill/commit/b72ccc35934165b4bad112b947ca5af064aab26f)) +* nested template literals in step inputs, and unresolvable $args tags ([#10856](https://github.com/windmill-labs/windmill/issues/10856)) ([8f349c0](https://github.com/windmill-labs/windmill/commit/8f349c032a0d75fc3350292075e5050a030f6166)) +* pre-fill the test panel JSON args editor and align its placeholder ([#10871](https://github.com/windmill-labs/windmill/issues/10871)) ([fb82f36](https://github.com/windmill-labs/windmill/commit/fb82f36e6d6492dd0740984d8d78ea4eaa30361e)) +* reject a prefixed error_handler_path on triggers ([#10847](https://github.com/windmill-labs/windmill/issues/10847)) ([d334831](https://github.com/windmill-labs/windmill/commit/d33483173526a3b352d2829ac8a2e1e229cc1127)) +* unify billable seat counting and prevent fork subscriptions ([#10818](https://github.com/windmill-labs/windmill/issues/10818)) ([7dd88c4](https://github.com/windmill-labs/windmill/commit/7dd88c470caee5f095dc240667aa7550c55696bc)) + ## [1.798.1](https://github.com/windmill-labs/windmill/compare/v1.798.0...v1.798.1) (2026-08-27) diff --git a/CONTEXT.md b/CONTEXT.md index 6efb92b669..fbc2592cc9 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -36,3 +36,41 @@ _Avoid_: argument field, param **Expression input**: Any other place a property can be picked into: the loop iterator, skip and early-stop predicates, the retry condition, a branch predicate, timeout. Its prop picker opens in a popover from the connect button rather than taking a pane. _Avoid_: JS field, code input + +### Flow chat + +**Conversation**: +One thread of messages against one chat-enabled flow, with its own agent memory. A flow has +many; the chat shows one at a time. +_Avoid_: thread, session (that names an AI session, a different thing), chat (that names the surface) + +**Turn**: +One question and the answer to it: the run the question started, the handle that stops it, +and the rows it is writing. At most one per conversation, and the chat is held for its whole +length — from the moment the question takes the chat, before it has a job, until it is ended. +_Avoid_: request, exchange, message round + +**Transcript**: +The rows a conversation's chat holds. Not the conversation: it is the newest page plus +whatever older pages the reader has scrolled back through, so a question it cannot answer +from what it holds is one to ask the server rather than to guess at. +_Avoid_: history, messages (too easily read as "all of them") + +### Permissions + +**Member**: +A user or group granted a role on a folder, a group, or an item's extra ACL. The list of them is +"Members (n)" everywhere it is shown, and one is added with "Add member". +_Avoid_: participant, collaborator, owner, ACL entry, permission (that names the concept, not the people) + +**Role**: +The access level a member holds: viewer, writer or admin on a folder; member or admin on a group. +Viewers read, writers also edit, admins also manage the members. A group role of **manager** — +manages the group without belonging to it — is a legacy state the UI shows and can leave, but +offers no way to enter. +_Avoid_: permission level, access level, rank + +**Owner**: +Reserved for the path prefix that says where an item lives — `u/alice` or `f/team`. A folder's +`owners` column in the database is its admin members; call those admins, never owners, in the UI. +_Avoid_: using "owner" for a folder admin diff --git a/Dockerfile b/Dockerfile index 8ecee96725..0fec95b95a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,6 +73,8 @@ COPY /backend/oauth_connect.json /backend/oauth_connect.json COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh COPY /system_prompts/auto-generated /system_prompts/auto-generated +# The flow chat imports the chat SDK's source (svelte.config.js alias `windmill-chat`). +COPY /chat-sdk/src /chat-sdk/src RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ @@ -141,9 +143,9 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 ARG KUBECTL_VERSION=1.36.2 -ARG HELM_VERSION=3.21.2 +ARG HELM_VERSION=3.21.4 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte -ARG GO_VERSION=1.26.0 +ARG GO_VERSION=1.26.8 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true @@ -250,8 +252,8 @@ RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_r RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ +RUN curl -sL https://deb.nodesource.com/setup_24.x | bash - +RUN apt-get -y update && apt-get install -y --no-install-recommends curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* # go build is slower the first time it is ran, so we prewarm it in the build @@ -299,7 +301,7 @@ RUN bun install -g windmill-cli \ RUN curl -fsSL https://claude.ai/install.sh | bash \ && cp /root/.local/share/claude/versions/* /usr/bin/claude -COPY --from=php:8.3.30-cli-trixie /usr/local/bin/php /usr/bin/php +COPY --from=php:8.3.33-cli-trixie /usr/local/bin/php /usr/bin/php COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled diff --git a/ai_evals/README.md b/ai_evals/README.md index ee8f730fc3..603f28e5f9 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -175,6 +175,10 @@ the decrypted value, exactly as against a real backend. The chat's read path pas Seed a recognizable secret (the existing fixture uses `sk_live_do_not_leak_me`) and assert it via `valueExcludes` to catch a leak. +`toolExpect.toolCallArgs` entries support `sharedByAtLeast: `: at least `n` recorded +calls to that tool must carry the same non-blank string in the field. Use it for calls that +have to share an identifier, like two test runs of one chat conversation. + `toolExpect.toolCallArgs` entries additionally support `fieldMustBeAbsent: true`: no recorded call to that tool may pass the field at all (an explicit `null` counts as passing it). Use it for partial-update tools, where supplying a field the model could diff --git a/ai_evals/adapters/frontend/backendPreview.ts b/ai_evals/adapters/frontend/backendPreview.ts index 57e1cfdf2a..8c1837c808 100644 --- a/ai_evals/adapters/frontend/backendPreview.ts +++ b/ai_evals/adapters/frontend/backendPreview.ts @@ -1,5 +1,5 @@ -import { randomUUID } from 'node:crypto' import type { BackendValidationSettings } from '../../core/backendValidation' +import { buildWorkspaceId } from './workspaceId' interface CompletedJobResultMaybe { completed: boolean @@ -24,7 +24,6 @@ export interface CompletedPreviewJob { const tokenCache = new Map>() const sharedWorkspaceQueue = new Map>() const managedSharedWorkspacePrefixes = ['f/evals/'] -const DEFAULT_WORKSPACE_PREFIX = 'ai-evals' export class BackendPreviewClient { constructor(private readonly settings: BackendValidationSettings) {} @@ -441,16 +440,6 @@ async function withSharedWorkspaceLock(workspaceId: string, body: () => Promi } } -function buildWorkspaceId(caseId: string, attempt: number): string { - const caseSlug = caseId - .toLowerCase() - .replace(/[^a-z0-9-]+/g, '-') - .replace(/^-+|-+$/g, '') - .slice(0, 30) - const suffix = randomUUID().slice(0, 8) - return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || 'case'}-a${attempt}-${suffix}` -} - function extractFolderName(path: string): string | null { if (!path.startsWith('f/')) { return null diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 4501f24810..2e29b9064d 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -132,6 +132,10 @@ export async function runEval( setToolStatus: () => {}, removeToolStatus: () => {}, isPlanModeActive, + // Accepts the run form exactly as the model prefilled it: there is nobody here to + // edit the arguments, so a case can assert what the model proposed but never how + // it reacts to the user changing something. + requestRunArgs: async (_toolId, form) => form.args, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 5c682be5eb..90df0cf870 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -7,12 +7,13 @@ import type { ListableApp, ListableResource, ListableVariable, + Resource, Script } from '../../../frontend/src/lib/gen' import type { + DataMetric, DataTableTables, DataTableTableSchema, - EndpointTool, GetDraftForUserResponse, GetOwnDraftResponse, ListDraftsResponse, @@ -47,12 +48,16 @@ export interface BenchmarkWorkspaceFlow { export interface BenchmarkWorkspaceApp { path: string summary: string + /** Defaults to true. Set false for a drag-and-drop app, which the chat can list + * and read but has no tool to edit — its value is a grid, not files. */ + rawApp?: boolean value: { - files: Record - runnables: Record + files?: Record + runnables?: Record data?: unknown policy?: unknown custom_path?: unknown + [key: string]: unknown } } @@ -81,8 +86,17 @@ export interface BenchmarkWorkspaceAiProvider { isDefault?: boolean } +/** A plain (non-AI) resource of the benchmark workspace, for cases about referencing a + * credential — passing one as a run argument, say. `value` is what `get_resource` returns. */ +export interface BenchmarkWorkspaceResource { + path: string + resource_type: string + value?: Record + description?: string +} + export interface BenchmarkWorkspaceJob { - /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ + /** Stable id so a case prompt can reference a specific run (e.g. for get_run). */ id?: string jobKind?: CompletedJob['job_kind'] scriptPath?: string @@ -90,6 +104,8 @@ export interface BenchmarkWorkspaceJob { label?: string success?: boolean logs?: string + args?: Record + result?: unknown } export interface BenchmarkWorkspaceRunnables { @@ -98,7 +114,11 @@ export interface BenchmarkWorkspaceRunnables { apps?: BenchmarkWorkspaceApp[] variables?: BenchmarkWorkspaceVariable[] aiProviders?: BenchmarkWorkspaceAiProvider[] + resources?: BenchmarkWorkspaceResource[] datatables?: BenchmarkDatatableSeed[] + /** DuckLake catalog names, as `list_ducklakes` reports them. */ + ducklakes?: string[] + dataMetrics?: DataMetric[] jobs?: BenchmarkWorkspaceJob[] } @@ -145,7 +165,7 @@ export function registerBenchmarkWorkspaceRunnables( ...runnables, datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined }) - // Seed any fixture jobs so list_runs / get_job_logs have data to return. + // Seed any fixture jobs so list_runs / get_run have data to return. for (const seed of runnables.jobs ?? []) { createBenchmarkCompletedJob({ workspace, @@ -155,7 +175,9 @@ export function registerBenchmarkWorkspaceRunnables( scriptPath: seed.scriptPath, createdBy: seed.createdBy, label: seed.label, - logs: seed.logs + logs: seed.logs, + args: seed.args, + result: seed.result }) } } @@ -284,15 +306,71 @@ export function listBenchmarkAiProviderResources(workspace: string): ListableRes })) } -/** The value of a seeded AI provider resource. Only the endpoint fields are modelled — a key is - * never needed, because no eval run calls the provider through this resource. */ +/** Plain seeded resources of a benchmark workspace, shaped like `ResourceService.listResource` + * rows. Null when the workspace is not a benchmark one. */ +export function listBenchmarkPlainResources(workspace: string): ListableResource[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + return null + } + return (runnables.resources ?? []).map((seed) => ({ + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: null, + is_oauth: false, + is_linked: false, + is_refreshed: false, + extra_perms: {}, + edited_at: BENCHMARK_TIMESTAMP + })) +} + +/** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both + * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI + * providers too, and a case that lists resources and then reads one by path would otherwise get + * a row it cannot fetch. */ +export function getBenchmarkResource(workspace: string, path: string): Resource | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const seed = runnables?.resources?.find((entry) => entry.path === path) + if (seed) { + return { + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: seed.value ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource + } + const provider = runnables?.aiProviders?.find((entry) => entry.path === path) + if (!provider) { + return null + } + return { + workspace_id: workspace, + path: provider.path, + resource_type: provider.kind, + value: getBenchmarkResourceValue(workspace, path) ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource +} + +/** The value of a seeded resource. For an AI provider only the endpoint fields are modelled — a + * key is never needed, because no eval run calls the provider through this resource. */ export function getBenchmarkResourceValue( workspace: string, path: string ): Record | null { - const seed = benchmarkWorkspaceRunnables - .get(workspace) - ?.aiProviders?.find((entry) => entry.path === path) + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const plain = runnables?.resources?.find((entry) => entry.path === path) + if (plain) { + return plain.value ?? {} + } + const seed = runnables?.aiProviders?.find((entry) => entry.path === path) if (!seed) { return null } @@ -414,6 +492,33 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { return job.logs ?? '' } +/** + * Mirror `JobService.getFlowAllResults`, which get_run calls for the execution + * tree. Fixture jobs are single runs with no steps, so only the root entry. + */ +export function getBenchmarkFlowAllResults(workspace: string, jobId: string) { + const job = getBenchmarkCompletedJob(workspace, jobId) + if (!job) { + throw new Error(`Job "${jobId}" not found in benchmark workspace`) + } + return { + entries: [ + { + job_id: jobId, + label: 'Flow', + kind: job.job_kind ?? 'script', + depth: 0, + sibling_index: 1, + sibling_count: 1, + status: job.success ? 'success' : 'failure', + success: job.success + } + ], + truncated: false, + scope_filtered: false + } +} + // ============= Drafts (per-user, DB-backed in production) ============= /** @@ -575,6 +680,27 @@ export function listBenchmarkDatatables(workspace: string): DataTableTables[] | })) } +// ============= DuckLake catalogs and declared metrics ============= + +/** Seeded DuckLake names, or `null` for a non-benchmark workspace. */ +export function listBenchmarkDucklakes(workspace: string): string[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + return runnables ? (runnables.ducklakes ?? []) : null +} + +/** + * Seeded metric declarations, or `null` for a non-benchmark workspace. + * + * The `table` / `path_prefix` filters are ignored: which rows a filter selects is + * `canonical_table_path`'s business and is pinned by `ducklakeTools.test.ts`. + * Re-deriving it here would give the eval its own copy of that spec to drift from, + * and the case this serves measures whether the model reaches for the tool at all. + */ +export function listBenchmarkDataMetrics(workspace: string): DataMetric[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + return runnables ? (runnables.dataMetrics ?? []) : null +} + export function getBenchmarkDatatableSchema(input: { workspace: string datatableName: string @@ -742,6 +868,29 @@ export function runBenchmarkFlowByPath(input: { }) } +/** + * Mirror `JobService.runFlowPreview` for benchmark workspaces, including the server's + * refusal of a chat-enabled flow run that names no conversation (`memory_id`). + */ +export function runBenchmarkFlowPreview(input: { + workspace: string + memoryId?: string + requestBody?: { path?: string; value?: { chat_input_enabled?: boolean }; args?: unknown } +}): string { + if (input.requestBody?.value?.chat_input_enabled && !input.memoryId) { + throw new Error('Bad request: memory_id is required for chat-enabled flows') + } + const args = (input.requestBody?.args ?? {}) as Record + return createBenchmarkCompletedJob({ + workspace: input.workspace, + jobKind: 'flowpreview', + success: true, + args, + result: { path: input.requestBody?.path, args, mocked: true }, + logs: 'Mock benchmark flow preview completed successfully.' + }) +} + export function previewBenchmarkSchedule(input: { requestBody?: Record }): Record { @@ -848,7 +997,7 @@ function buildBenchmarkListableApp(app: BenchmarkWorkspaceApp): ListableApp { extra_perms: {}, edited_at: BENCHMARK_TIMESTAMP, execution_mode: 'viewer', - raw_app: true + raw_app: app.rawApp ?? true } } @@ -866,123 +1015,10 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { execution_mode: 'viewer', extra_perms: {}, custom_path: app.value.custom_path as string | undefined, - raw_app: true + raw_app: app.rawApp ?? true } } -// ============= API endpoint catalog (McpService.listMcpTools + raw fetch) ============= -// The global chat's API catalog tools list endpoints via McpService and execute -// them with a plain relative fetch('/api/...'), which has no meaning in the -// vitest environment. A representative slice of the real catalog is served here, -// and `handleBenchmarkApiFetch` answers the executed calls. - -const BENCHMARK_MCP_TOOLS: EndpointTool[] = [ - { - name: 'listWorkers', - description: 'List workers', - instructions: 'List all workers with their last ping and job counts.', - path: '/workers/list', - method: 'GET', - query_params_schema: { - type: 'object', - properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } - } - }, - { - name: 'listQueue', - description: 'List queued jobs', - instructions: '', - path: '/w/{workspace}/jobs/queue/list', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' } }, - required: ['workspace'] - } - }, - { - name: 'getJob', - description: 'get job', - instructions: '', - path: '/w/{workspace}/jobs_u/get/{id}', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string', format: 'uuid' } }, - required: ['workspace', 'id'] - }, - query_params_schema: { - type: 'object', - properties: { - no_logs: { type: 'boolean' }, - no_code: { type: 'boolean' }, - approval_token: { type: 'string' } - }, - required: [] - } - }, - { - name: 'runScriptByPath', - description: 'Run the deployed version of a script by path', - instructions: '', - path: '/w/{workspace}/jobs/run/p/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', properties: {} } - }, - { - name: 'runFlowByPath', - description: 'Run the deployed version of a flow by path', - instructions: '', - path: '/w/{workspace}/jobs/run/f/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', properties: {} } - }, - // Draft-covered endpoints, present so steering cases exercise the guard the - // way production does (hidden from search, refused at call time). - { - name: 'getScriptByPath', - description: 'Get a script by path', - instructions: '', - path: '/w/{workspace}/scripts/get/p/{path}', - method: 'GET' - }, - { - name: 'createFlow', - description: 'Create a flow', - instructions: '', - path: '/w/{workspace}/flows/create', - method: 'POST' - }, - { - name: 'deleteSchedule', - description: 'Delete a schedule', - instructions: '', - path: '/w/{workspace}/schedules/delete/{path}', - method: 'DELETE' - }, - { - name: 'getVariable', - description: 'Get a variable', - instructions: '', - path: '/w/{workspace}/variables/get/{path}', - method: 'GET' - } -] - -export function listBenchmarkMcpTools(): EndpointTool[] { - return BENCHMARK_MCP_TOOLS -} - /** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over * relative `/api/...` fetches, which have no origin here, so without these the * hub tools throw and no case can exercise hub reuse. Serving fixtures rather @@ -1142,26 +1178,6 @@ const BENCHMARK_WORKERS = [ } ] -const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/ -const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/ - -/** `executeEndpoint` sends a JSON string; anything else means no args were supplied. */ -function parseBenchmarkRequestBody( - body: BodyInit | null | undefined -): Record | undefined { - if (typeof body !== 'string') { - return undefined - } - try { - const parsed = JSON.parse(body) - return typeof parsed === 'object' && parsed !== null - ? (parsed as Record) - : undefined - } catch { - return undefined - } -} - /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. * Any other relative fetch must keep its normal (non-benchmark) behavior — * intercepting it with a synthetic 404 sends the model into retry loops. */ @@ -1173,17 +1189,13 @@ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] return ( path === '/api/workers/list' || - BENCHMARK_JOB_GET_PATH.test(path) || - BENCHMARK_RUN_BY_PATH.test(path) || - /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || path === '/api/embeddings/query_hub_scripts' || path.startsWith('/api/scripts/hub/get_full/') || BENCHMARK_AI_MODELS_PATH.test(path) ) } -/** Answer a relative `/api/...` fetch — from the API catalog executor, or from the - * chat's hub tools. */ +/** Answer the relative `/api/...` fetches no mocked service covers. */ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Response { const path = url.split('?')[0] if (path === '/api/workers/list') { @@ -1200,39 +1212,6 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon ?.aiProviders?.find((entry) => entry.path === resourcePath) return Response.json({ data: (seed?.models ?? []).map((id) => ({ id })) }) } - if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { - return Response.json([]) - } - const jobGet = BENCHMARK_JOB_GET_PATH.exec(path) - if (jobGet) { - const id = decodeURIComponent(jobGet[2]) - const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id) - if (!job) { - return Response.json({ error: `Job not found for "${id}"` }, { status: 404 }) - } - // The real endpoint lets a caller drop the bulky fields. Ignoring that here would - // size the model's context off a payload it explicitly asked to shrink. - const query = new URLSearchParams(url.split('?')[1] ?? '') - if (query.get('no_logs') === 'true') { - delete job.logs - } - if (query.get('no_code') === 'true') { - delete job.raw_code - } - return Response.json(job) - } - const runByPath = BENCHMARK_RUN_BY_PATH.exec(path) - if (runByPath) { - const workspace = decodeURIComponent(runByPath[1]) - const runnablePath = decodeURIComponent(runByPath[3]) - const args = parseBenchmarkRequestBody(init?.body) - // The real endpoint answers with the bare job id as text, not JSON. - return new Response( - runByPath[2] === 'f' - ? runBenchmarkFlowByPath({ workspace, path: runnablePath, args }) - : runBenchmarkScriptByPath({ workspace, path: runnablePath, args }) - ) - } if (path === '/api/embeddings/query_hub_scripts') { const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' return Response.json(searchBenchmarkHubScripts(text)) diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts index d9cfa08bf4..85bd1f3320 100644 --- a/ai_evals/adapters/frontend/mockBackendApi.test.ts +++ b/ai_evals/adapters/frontend/mockBackendApi.test.ts @@ -1,83 +1,18 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test' -import { - createBenchmarkCompletedJob, - getBenchmarkCompletedJob, - handleBenchmarkApiFetch, - hasBenchmarkApiHandler, - listBenchmarkMcpTools, - resetBenchmarkMockBackend, - registerBenchmarkWorkspaceRunnables -} from './mockBackend' +import { describe, expect, it } from 'bun:test' +import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' -const WORKSPACE = 'benchmark-api-ws' +// `list_workers`'s client call is not faked, so its request really does leave as a relative +// `/api/...` url that node's fetch cannot resolve. Without this route the tool throws instead +// of answering. +describe('benchmark API fetch handlers', () => { + it('lists workers, the way list_workers reaches them', async () => { + const url = '/api/workers/list?per_page=100' -// A catalog entry with no fetch handler is a dead end: the catalog executor builds a -// relative `/api/...` url, the stub declines it, and node's fetch throws on the relative -// url instead of returning a result the model can act on. Mutating entries are reachable -// too — the eval runners define no `requestConfirmation`, so `call_api_endpoint` executes -// unconfirmed. -describe('benchmark API catalog', () => { - beforeEach(() => resetBenchmarkMockBackend()) - afterEach(() => resetBenchmarkMockBackend()) + expect(hasBenchmarkApiHandler(url)).toBe(true) + const body = await handleBenchmarkApiFetch(url).json() - it('answers every endpoint it advertises', () => { - const unanswered = listBenchmarkMcpTools() - .map((tool) => - `/api${tool.path.replace('{workspace}', WORKSPACE)}`.replace(/\{[^}]+\}/g, 'x') - ) - .filter((url) => !hasBenchmarkApiHandler(url)) - - // The draft-covered entries are refused by name before any fetch, so they are - // advertised without a handler on purpose. - expect(unanswered).toEqual([ - `/api/w/${WORKSPACE}/scripts/get/p/x`, - `/api/w/${WORKSPACE}/flows/create`, - `/api/w/${WORKSPACE}/schedules/delete/x`, - `/api/w/${WORKSPACE}/variables/get/x` - ]) - }) - - it('runs a deployed script by path, the way call_api_endpoint reaches it', async () => { - registerBenchmarkWorkspaceRunnables(WORKSPACE, { - scripts: [ - { - path: 'f/evals/greet', - summary: 'Greet', - language: 'bun', - content: 'export async function main() {}' - } - ] - }) - - const res = handleBenchmarkApiFetch( - `/api/w/${WORKSPACE}/jobs/run/p/${encodeURIComponent('f/evals/greet')}`, - { method: 'POST', body: JSON.stringify({ name: 'ada' }) } - ) - - expect(res.status).toBe(200) - const job = getBenchmarkCompletedJob(WORKSPACE, (await res.text()).trim()) - expect(job).toMatchObject({ success: true, args: { name: 'ada' } }) - }) - - it('serves a recorded job so a model can check the run it just started', async () => { - const id = createBenchmarkCompletedJob({ - workspace: WORKSPACE, - jobKind: 'preview', - result: 'Hello, World!' - }) - - const res = handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/${id}`) - - expect(res.status).toBe(200) - expect(await res.json()).toMatchObject({ - id, - success: true, - result: 'Hello, World!' - }) - }) - - it('404s an unknown job id instead of letting the fetch fall through', () => { - expect(hasBenchmarkApiHandler(`/api/w/${WORKSPACE}/jobs_u/get/missing`)).toBe(true) - expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404) + // The tool counts the list and maps over it, so it needs a bare array, not an envelope. + expect(Array.isArray(body)).toBe(true) + expect(body[0]).toMatchObject({ worker: expect.any(String), worker_group: expect.any(String) }) }) }) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index e59ec11ad9..0c8a22a987 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -5,8 +5,8 @@ import { mkdir, writeFile } from 'fs/promises' import { dirname, resolve } from 'path' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' -// The API catalog executor issues relative fetch('/api/...') calls, which have -// no meaning in the vitest environment — serve the ones the benchmark handles. +// Some tools reach the backend by relative fetch('/api/...'), which has no meaning +// in the vitest environment — serve the ones the benchmark handles. // Every other relative fetch keeps its normal behavior (it fails the same way // it does without this stub) so unrelated tools see an unchanged environment. // The frontend builds API URLs from location.origin (fetchAvailableModels does), and node has no @@ -62,6 +62,7 @@ vi.mock('$lib/gen', async () => { getBenchmarkDatatableSchema, getBenchmarkDraftForUser, getBenchmarkFlowByPath, + getBenchmarkFlowAllResults, getBenchmarkJobLogs, getBenchmarkOwnDraft, getBenchmarkScriptByHash, @@ -70,10 +71,14 @@ vi.mock('$lib/gen', async () => { getBenchmarkResourceValue, getBenchmarkVariableByPath, hasBenchmarkWorkspace, + getBenchmarkResource, listBenchmarkAiProviderResources, + listBenchmarkPlainResources, listBenchmarkApps, listBenchmarkDatatables, + listBenchmarkDataMetrics, listBenchmarkDrafts, + listBenchmarkDucklakes, listBenchmarkFlows, listBenchmarkJobs, listBenchmarkScripts, @@ -84,9 +89,10 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, + runBenchmarkFlowPreview, + runBenchmarkScriptByPath, runBenchmarkScriptPreview, - updateBenchmarkDraft, - listBenchmarkMcpTools + updateBenchmarkDraft } = await import('./mockBackend') function wrapService(target: T, overrides: Record): T { @@ -277,6 +283,26 @@ vi.mock('$lib/gen', async () => { } return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody }) }, + runScriptByPath: async (data: { + workspace: string + path: string + requestBody?: Record + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkScriptByPath({ + workspace: data.workspace, + path: data.path, + args: data.requestBody + }) + : actual.JobService.runScriptByPath(data), + runFlowPreview: async (data: { + workspace: string + memoryId?: string + requestBody?: { path?: string; value?: { chat_input_enabled?: boolean }; args?: unknown } + }) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkFlowPreview(data) + : actual.JobService.runFlowPreview(data as any), runFlowByPath: async (data: { workspace: string path: string @@ -310,7 +336,11 @@ vi.mock('$lib/gen', async () => { getJobLogs: async (data: { workspace: string; id: string }) => hasBenchmarkWorkspace(data.workspace) ? getBenchmarkJobLogs(data.workspace, data.id) - : actual.JobService.getJobLogs(data) + : actual.JobService.getJobLogs(data), + getFlowAllResults: async (data: { workspace: string; id: string }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkFlowAllResults(data.workspace, data.id) + : actual.JobService.getFlowAllResults(data) }), WorkspaceService: wrapService(actual.WorkspaceService, { getCopilotInfo: async (data: { workspace: string }) => @@ -321,6 +351,10 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? (listBenchmarkDatatables(data.workspace) ?? []) : actual.WorkspaceService.listDataTableTables(data), + listDucklakes: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) + ? (listBenchmarkDucklakes(data.workspace) ?? []) + : actual.WorkspaceService.listDucklakes(data), getDataTableTableSchema: async (data: { workspace: string datatableName: string @@ -336,6 +370,12 @@ vi.mock('$lib/gen', async () => { }) : actual.WorkspaceService.getDataTableTableSchema(data) }), + DataMetricService: wrapService(actual.DataMetricService, { + listDataMetrics: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) + ? { metrics: listBenchmarkDataMetrics(data.workspace) ?? [] } + : actual.DataMetricService.listDataMetrics(data) + }), ScheduleService: wrapService(actual.ScheduleService, { existsSchedule: async (data: { workspace: string; path: string }) => hasBenchmarkWorkspace(data.workspace) ? false : actual.ScheduleService.existsSchedule(data), @@ -359,18 +399,24 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkResourceValue(data.workspace, data.path)) : actual.ResourceService.existsResource(data), - // Only AI provider resources are modelled: they are what an AI agent step references. listResource: async (data: { workspace: string; resourceType?: string }) => { if (!hasBenchmarkWorkspace(data.workspace)) { return actual.ResourceService.listResource(data) } - const seeded = listBenchmarkAiProviderResources(data.workspace) ?? [] + const seeded = [ + ...(listBenchmarkAiProviderResources(data.workspace) ?? []), + ...(listBenchmarkPlainResources(data.workspace) ?? []) + ] const wanted = data.resourceType?.split(',') return wanted ? seeded.filter((r) => wanted.includes(r.resource_type)) : seeded }, getResource: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + const resource = getBenchmarkResource(data.workspace, data.path) + if (!resource) { + throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + } + return resource } return actual.ResourceService.getResource(data) }, @@ -387,12 +433,6 @@ vi.mock('$lib/gen', async () => { queryResourceTypes: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data) }), - McpService: wrapService(actual.McpService, { - listMcpTools: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? listBenchmarkMcpTools() - : actual.McpService.listMcpTools(data) - }), VariableService: wrapService(actual.VariableService, { existsVariable: async (data: { workspace: string; path: string }) => hasBenchmarkWorkspace(data.workspace) diff --git a/ai_evals/adapters/frontend/windmillBackend.ts b/ai_evals/adapters/frontend/windmillBackend.ts index 2247d8e5d5..bf483aa1f4 100644 --- a/ai_evals/adapters/frontend/windmillBackend.ts +++ b/ai_evals/adapters/frontend/windmillBackend.ts @@ -1,9 +1,8 @@ -import { randomUUID } from "node:crypto"; import type { WindmillBackendSettings } from "../../core/windmillBackendSettings"; +import { buildWorkspaceId } from "./workspaceId"; const tokenCache = new Map>(); const sharedWorkspaceQueue = new Map>(); -const DEFAULT_WORKSPACE_PREFIX = "ai-evals"; export class WindmillBackendClient { constructor(private readonly settings: WindmillBackendSettings) {} @@ -179,16 +178,6 @@ async function withSharedWorkspaceLock( } } -function buildWorkspaceId(caseId: string, attempt: number): string { - const caseSlug = caseId - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 30); - const suffix = randomUUID().slice(0, 8); - return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || "case"}-a${attempt}-${suffix}`; -} - async function expectOk(response: Response, context: string): Promise { if (response.ok) { return; diff --git a/ai_evals/adapters/frontend/workspaceId.test.ts b/ai_evals/adapters/frontend/workspaceId.test.ts new file mode 100644 index 0000000000..3573a9e51a --- /dev/null +++ b/ai_evals/adapters/frontend/workspaceId.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "bun:test"; +import { buildWorkspaceId } from "./workspaceId"; + +describe("buildWorkspaceId", () => { + // `workspace.proper_id` rejects `--`, which a case id can carry itself and + // which truncating a slug on a hyphen produces once the suffix adds its own. + // One id per shape: cut landing on a hyphen, cut landing mid-word, no cut, and + // a doubled hyphen no cut ever reaches. + it("stays within the id length cap and the proper_id format", () => { + for (const caseId of [ + "global-test6-secret-variable-draft", + "global-test23-datatable-query-select", + "short", + "global--test-foo", + ]) { + const id = buildWorkspaceId(caseId, 1); + expect(id.length).toBeLessThanOrEqual(50); + expect(id).toMatch(/^\w+(-\w+)*$/); + } + }); +}); diff --git a/ai_evals/adapters/frontend/workspaceId.ts b/ai_evals/adapters/frontend/workspaceId.ts new file mode 100644 index 0000000000..9545c06a7d --- /dev/null +++ b/ai_evals/adapters/frontend/workspaceId.ts @@ -0,0 +1,22 @@ +import { randomUUID } from "node:crypto"; + +const DEFAULT_WORKSPACE_PREFIX = "ai-evals"; + +// A workspace id must be at most 50 characters AND match `^\w+(-\w+)*$` +// (`workspace.proper_id`), so the case slug yields to the random suffix that +// makes the id unique, and no hyphen may end up doubled — neither one already in +// the case id nor one a truncation leaves for the suffix to follow. +const MAX_WORKSPACE_ID_LENGTH = 50; + +export function buildWorkspaceId(caseId: string, attempt: number): string { + const caseSlug = caseId + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/-{2,}/g, "-") + .replace(/^-+|-+$/g, ""); + const suffix = `-a${attempt}-${randomUUID().slice(0, 8)}`; + const head = `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || "case"}`; + return `${head + .slice(0, MAX_WORKSPACE_ID_LENGTH - suffix.length) + .replace(/-+$/, "")}${suffix}`; +} diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 21f80bef7e..b17d37cb29 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -889,13 +889,13 @@ draftCountExactly: 0 toolExpect: requiredToolsUsed: - - get_job_logs + - get_run forbiddenToolsUsed: - deploy_workspace_item - delete_workspace_item - write_script toolCallArgs: - - tool: get_job_logs + - tool: get_run field: id stringIncludesAnyOf: - 01920000-0000-7000-8000-0000000000f1 @@ -906,6 +906,34 @@ - fetches the logs for the requested job id - explains the failure from the returned logs (connection refused to the upstream API) +- id: global-run-args-and-result + prompt: |- + What was the run 01920000-0000-7000-8000-0000000000f2 called with, and what did it return? + initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_run + forbiddenToolsUsed: + - test_run_script + - run_script + - deploy_workspace_item + toolCallArgs: + - tool: get_run + field: id + stringIncludesAnyOf: + - 01920000-0000-7000-8000-0000000000f2 + # Read-only, so no draft for the global judge to score — validated on tool use + # and the deterministic argument check, like the neighbouring run cases. + skipJudge: true + judgeChecklist: + - reports the arguments the run was called with (region emea, 12 recipients) + - reports what the run returned (12 sent, 3 skipped) + - does not start a new run to find out + # --- Page navigation (open_page) --- # The assistant should take the user to a Windmill page (Runs/Schedules) with the # right filters via open_page, rather than describing where to click or dumping the @@ -1891,10 +1919,10 @@ - when the lookup fails, tells the user instead of inventing table names - does not write scripts or resources to answer a read-only question -# --- API catalog (search_api_endpoints / call_api_get / call_api_endpoint) --- -# The harness serves the catalog and the executed calls itself (mock -# listMcpTools + benchmark fetch handlers in adapters/frontend), so these cases -# do not require an mcp-enabled eval backend. +# --- Dedicated tools for workspace, run and instance state --- +# The harness serves the worker read itself (a benchmark fetch handler in +# adapters/frontend). The stale `api-catalog` in the id below is kept so +# results stay comparable across benchmark runs. - id: global-test30-api-catalog-workers prompt: |- @@ -1906,23 +1934,40 @@ draftCountExactly: 0 toolExpect: requiredToolsUsed: - - search_api_endpoints - - call_api_get + - list_workers forbiddenToolsUsed: - - call_api_endpoint - write_script - deploy_workspace_item - toolCallArgs: - - tool: call_api_get - field: name - stringIncludesAnyOf: - - listWorkers # Read-only workspace inspection produces no draft; validate via tool use. skipJudge: true judgeChecklist: - - discovers the workers endpoint through the API catalog instead of guessing or fabricating + - reads worker state through list_workers instead of guessing or fabricating - reports worker status from the returned data +- id: global-test37-ducklake-declared-measure + prompt: |- + We track orders in the main ducklake. Write me a duckdb script that reports total + revenue by month. Keep it as a draft, don't deploy it. + initial: ai_evals/fixtures/frontend/global/initial/ducklake_orders_metrics.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + toolExpect: + requiredToolsUsed: + - list_data_metrics + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + # The judge runs: the point is not that the tool was called but that the number it + # describes is the declared one. `revenue` excludes test rows, so an aggregate that + # reproduces it without the filter is plausible, runnable and wrong. + judgeChecklist: + - totals revenue with the declared sum over the amount column rather than an invented aggregate over a guessed column + - excludes test orders from the total, as the declared revenue measure does + - groups by month using the declared order_month expression over order_date + - does not introduce column names absent from the declarations + - id: global-test31-draft-test-run-not-deployed prompt: |- Update `f/evals/global/format_greeting` so the provided name is uppercased in the greeting, then run it with name "ada" to check it works. @@ -1942,13 +1987,11 @@ requiredToolsUsed: - test_run_script forbiddenToolsUsed: - - call_api_endpoint - deploy_workspace_item - delete_workspace_item # The judge only sees the drafts artifact and cannot observe runs, so it always # docks the prompt's "run it" requirement — validate deterministically instead: - # draft content via valueIncludes, the test run via toolExpect (test_run_script - # required, call_api_endpoint forbidden). + # draft content via valueIncludes, the test run via toolExpect. skipJudge: true judgeChecklist: - creates an AI draft of f/evals/global/format_greeting with the name uppercased in the greeting @@ -1966,13 +2009,172 @@ requiredToolsUsed: - delete_workspace_item forbiddenToolsUsed: - - call_api_endpoint - - search_api_endpoints - write_script # Deletion produces no draft; validate via tool use. skipJudge: true judgeChecklist: - - deletes the deployed script via delete_workspace_item rather than a raw API endpoint + - deletes the deployed script via delete_workspace_item + +- id: global-test33-run-deployed-script-with-form + prompt: |- + Run the deployed script `f/evals/global/format_greeting` for me with the name "ada". + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_script + - write_script + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_script + field: args.name + stringIncludesAnyOf: + - ada + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed script through run_script rather than a preview test run + - passes the name "ada" so the confirmation form comes up prefilled + +- id: global-test34-run-with-secret-from-variable + prompt: |- + Run the deployed `f/evals/global/billing_sync` for the account `acme` — use the billing + API token we already keep in the workspace. + initial: ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json + runtime: + maxTurns: 10 + # A session chat is where the run card has a preview pane beside it; run_script + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + # A secret argument is filled by naming the variable that holds it: the value stays in + # the variable and only its path travels. A literal reaches the job as a reference too, + # minted on the way in, but it stays in the tool call the model emitted. + toolCallArgs: + - tool: run_script + field: args.api_token + stringIncludesAnyOf: + - "$var:f/evals/global/stripe_api_token" + - tool: run_script + field: args.account + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - fills the secret argument with a reference to the existing workspace variable rather than a literal token + - passes the account "acme" + - does not invent or guess the token's value + +- id: global-test35-run-deployed-flow-with-form + prompt: |- + Run the deployed flow `f/evals/global/notify_customer` for me — the customer is `acme`. + initial: ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_flow + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_flow + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_flow + - write_flow + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_flow + field: args.customer + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed flow through run_flow rather than a preview test run + - passes the customer "acme" so the confirmation form comes up prefilled + +- id: global-test36-draft-flow-test-run-not-deployed + prompt: |- + Update the `calculate_total` step of `f/evals/global/process_invoice` so it applies 8% tax and + returns `subtotal`, `tax` and `total`, then run it to check it works. + Keep it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/process_invoice + toolExpect: + # A one-step flow is as well checked by running the step as the whole flow, so both + # count: what matters is that the run is against the draft. + requiredToolsAnyOf: + - [test_run_flow, test_run_step] + # The draft is what the user asked to check, and run_flow would run the deployed + # version instead — the edit would not be in what ran. + forbiddenToolsUsed: + - run_flow + - deploy_workspace_item + # The judge cannot observe runs, and the edit's content is already pinned by + # global-test5 on this fixture; what this case guards is where the run went. + skipJudge: true + judgeChecklist: + - creates an AI draft of f/evals/global/process_invoice applying 8% tax + - does not deploy or save the draft + +- id: global-test38-chat-flow-follow-up-same-conversation + prompt: |- + I want to check that my support chat flow `f/evals/global/support_chat` remembers what was said. + Test it: first send "My name is Ada", then send "What is my name?" as a follow-up in the same chat. + initial: ai_evals/fixtures/frontend/global/initial/support_chat_flow.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - test_run_flow + # A chat flow's memory lives in its conversation, so a follow-up only reaches the first + # turn's history when both test runs name the same conversation. + toolCallArgs: + - tool: test_run_flow + field: memory_id + sharedByAtLeast: 2 + forbiddenToolsUsed: + - run_flow + - deploy_workspace_item + # The judge cannot observe runs; what this case guards is the conversation the runs share. + skipJudge: true + judgeChecklist: + - test-runs the chat flow twice, the second message as a follow-up in the first run's conversation - id: global-undo-created-draft prompt: |- @@ -2365,3 +2567,74 @@ - the step uses the workspace's anthropic resource f/evals/global/anthropic_main - the model is the Opus one the user asked for, taken from the models that resource serves - the diff flow input reaches the agent + +# The failure this pins: passing a resource as `{"$res": ""}` (or as a bare path), which +# reaches the script unresolved because the backend only substitutes a string value that itself +# starts with `$res:`. The mock preview echoes args back and reports success, so nothing in the +# loop corrects a wrong shape — the arg form is the whole test. +- id: global-run-arg-resource-reference + prompt: |- + Run `f/evals/global/github_repo_stats` against the `windmill-labs/windmill` repo, passing our + GitHub credentials at `f/evals/global/github_main` as its `gh_auth` input, and tell me whether + it went through. + initial: ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - test_run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + toolCallArgs: + # Exact: the mock never resolves the reference, so a near-miss path like + # `$res:f/evals/global/github_main_backup` would otherwise pass. + - tool: test_run_script + field: args.gh_auth + stringEqualsAnyOf: + - "$res:f/evals/global/github_main" + # The judge only sees drafts, and this case makes none — the deliverable is the shape of the + # run argument, checked deterministically above. + skipJudge: true + judgeChecklist: + - runs the existing script rather than rewriting it + - passes the GitHub resource as the bare string $res:f/evals/global/github_main + +- id: global-drag-and-drop-app-not-editable + prompt: |- + Add a refresh button to the ops console app, and the same to the sales board app. + initial: ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json + runtime: + maxTurns: 12 + validate: + # One request, two apps, only one of them editable: the code app must come back with a + # draft and the drag-and-drop one must not. Refusing both, or editing both, fails here — + # which is what makes this a test of the distinction rather than of caution. + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/ops_console + forbiddenDrafts: + - type: app + path: f/evals/global/sales_board + toolExpect: + # Deliberately not constraining write_app_file/patch_app_file by argument: an entry there + # fails when its tool was never called, so naming both would fail on whichever the model + # did not pick. The draft assertions above cover the same ground, tool-agnostically. + forbiddenToolsUsed: + - init_app + - deploy_workspace_item + - delete_app_file + - delete_app_runnable + assistantExpect: + # A refusal leaves no draft for the judge to read, so the explanation is checked here. + # Only the app kind: substring tests cannot see paraphrase, and every wording of "I can't + # edit it" defeats a fixed list. + requiredMentionsAnyOf: + - - drag-and-drop + - drag and drop + - low-code + - no-code + skipJudge: true diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index c9a4f7830e..832a383a73 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -160,6 +160,13 @@ export interface ToolCallArgumentRule { field: string; stringStartsWithAnyOf?: string[]; stringMustNotStartWithAnyOf?: string[]; + /** + * Universal over calls: every recorded call to `tool` must carry `field` as + * exactly one of these strings. Use when a near-miss would still satisfy a + * prefix — a resource reference like `$res:f/a/b` shares its prefix with the + * wrong `$res:f/a/b_backup`, and the mock never resolves it to catch that. + */ + stringEqualsAnyOf?: string[]; /** * Case-insensitive "contains", existential over calls: at least one recorded * call to `tool` must have `field` containing one of these substrings. Other @@ -175,6 +182,13 @@ export interface ToolCallArgumentRule { * the point is that the model filled it in at all rather than what it said. */ nonEmpty?: boolean; + /** + * Existential over calls: at least this many recorded calls to `tool` carry the + * same non-blank string in `field`. Use when calls have to share an identifier — + * e.g. test runs that continue one conversation — while a retry with a rejected + * value in between is still acceptable. + */ + sharedByAtLeast?: number; /** * Universal over calls: no recorded call to `tool` may pass `field` at all. * For partial-update tools, where supplying a field the model could not have diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 9b606c1c85..1f7974e34e 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -228,6 +228,43 @@ describe("validateToolExpectations", () => { }); }); + // A resource reference shares its prefix with a wrong sibling path, and the mock + // never resolves it, so only exact matching separates the two. + it("rejects a resource reference whose path merely shares the prefix", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["test_run_script"], + toolCallDetails: [ + { + name: "test_run_script", + arguments: { args: { gh_auth: "$res:f/evals/global/github_main_backup" } }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [ + { + tool: "test_run_script", + field: "args.gh_auth", + stringEqualsAnyOf: ["$res:f/evals/global/github_main"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "test_run_script.args.gh_auth matches an accepted value", + passed: false, + details: + 'accepted values: $res:f/evals/global/github_main; values: "$res:f/evals/global/github_main_backup"', + }); + }); + // The whole point of the same-call rule: the per-field rules are existential over // calls, so two single-filter pages would satisfy them while never opening the // combined view the case asks for. @@ -359,6 +396,32 @@ describe("validateToolExpectations", () => { expect(nonEmptyCheck?.details).toContain("blank on 1 of 2"); }); + it("requires sharedByAtLeast calls to carry one value, not merely a value each", () => { + const run = (ids: (string | undefined)[]) => + validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: ids.length, + toolsUsed: ["test_run_flow"], + toolCallDetails: ids.map((memory_id) => ({ + name: "test_run_flow", + arguments: { path: "f/chat", memory_id }, + })), + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [{ tool: "test_run_flow", field: "memory_id", sharedByAtLeast: 2 }], + }, + }).find((c) => c.name.includes("is shared by at least 2 calls"))?.passed; + + expect(run(["a", "b"])).toBe(false); + expect(run(["a"])).toBe(false); + expect(run([undefined, undefined])).toBe(false); + expect(run(["rejected", "a", "a"])).toBe(true); + }); + it("passes nonEmpty when every call filled the field", () => { const checks = validateToolExpectations({ run: { diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 7150dab0c3..9864f33dcc 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -278,6 +278,20 @@ export function validateToolExpectations(input: { ); } + if (rule.stringEqualsAnyOf && rule.stringEqualsAnyOf.length > 0) { + const invalidValues = values.filter( + (value) => + typeof value !== "string" || !rule.stringEqualsAnyOf!.includes(value) + ); + checks.push( + check( + `${rule.tool}.${rule.field} matches an accepted value`, + invalidValues.length === 0, + `accepted values: ${rule.stringEqualsAnyOf.join(", ")}; values: ${summarizeToolValues(values)}` + ) + ); + } + if (rule.stringMustNotStartWithAnyOf && rule.stringMustNotStartWithAnyOf.length > 0) { const invalidValues = values.filter( (value) => @@ -306,6 +320,23 @@ export function validateToolExpectations(input: { ); } + if (rule.sharedByAtLeast !== undefined) { + const counts = new Map(); + for (const value of values) { + if (typeof value === "string" && value.trim().length > 0) { + counts.set(value, (counts.get(value) ?? 0) + 1); + } + } + const mostShared = Math.max(0, ...counts.values()); + checks.push( + check( + `${rule.tool}.${rule.field} is shared by at least ${rule.sharedByAtLeast} calls`, + mostShared >= rule.sharedByAtLeast, + `most calls sharing one value: ${mostShared}; values: ${summarizeToolValues(values)}` + ) + ); + } + if (rule.fieldMustBeAbsent) { // Anything other than `undefined` was supplied — an explicit `null` is the // model passing the field, not omitting it. diff --git a/ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json b/ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json new file mode 100644 index 0000000000..fbef7b3e3d --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/apps_code_and_drag_and_drop.json @@ -0,0 +1,30 @@ +{ + "user": { + "username": "admin", + "is_admin": true, + "folders": ["evals"], + "folders_read": ["evals"] + }, + "workspace": { + "apps": [ + { + "path": "f/evals/global/sales_board", + "summary": "Sales board", + "rawApp": false, + "value": { + "grid": [] + } + }, + { + "path": "f/evals/global/ops_console", + "summary": "Ops console", + "value": { + "files": { + "/App.tsx": "export default function App() {\n\treturn
Ops console
\n}\n" + }, + "runnables": {} + } + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json new file mode 100644 index 0000000000..ef166b0f9c --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/billing_sync_with_secret_arg.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "variables": [ + { + "path": "f/evals/global/stripe_api_token", + "value": "sk_live_do_not_leak_me", + "is_secret": true, + "description": "Token used by the billing sync job", + "labels": ["billing"] + } + ], + "scripts": [ + { + "path": "f/evals/global/billing_sync", + "summary": "Sync billing records", + "description": "Syncs billing records for one account, authenticating with an API token.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "account": { + "type": "string" + }, + "api_token": { + "type": "string", + "password": true, + "description": "API token to authenticate with" + } + }, + "required": ["account", "api_token"] + }, + "content": "export async function main(account: string, api_token: string) {\n return `synced ${account}`\n}\n" + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/ducklake_orders_metrics.json b/ai_evals/fixtures/frontend/global/initial/ducklake_orders_metrics.json new file mode 100644 index 0000000000..fba88c53d3 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/ducklake_orders_metrics.json @@ -0,0 +1,36 @@ +{ + "workspace": { + "ducklakes": ["main"], + "dataMetrics": [ + { + "script_path": "f/analytics/orders_pipeline", + "table_path": "main/main.orders", + "kind": "measure", + "name": "revenue", + "expr": "sum(amount)", + "filter": "not is_test" + }, + { + "script_path": "f/analytics/orders_pipeline", + "table_path": "main/main.orders", + "kind": "measure", + "name": "order_count", + "expr": "count(*)" + }, + { + "script_path": "f/analytics/orders_pipeline", + "table_path": "main/main.orders", + "kind": "dimension", + "name": "order_month", + "expr": "date_trunc('month', order_date)" + }, + { + "script_path": "f/analytics/orders_pipeline", + "table_path": "main/main.orders", + "kind": "dimension", + "name": "region", + "expr": "region" + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json new file mode 100644 index 0000000000..80e1b960c5 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "resources": [ + { + "path": "f/evals/global/github_main", + "resource_type": "github", + "description": "GitHub credentials", + "value": { "token": "$var:f/evals/global/github_token" } + } + ], + "scripts": [ + { + "path": "f/evals/global/github_repo_stats", + "summary": "Count open issues on a GitHub repository", + "description": "Reads the open issue count for a repository using GitHub credentials.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "gh_auth": { + "type": "object", + "format": "resource-github", + "description": "GitHub credentials" + }, + "repo": { + "type": "string", + "description": "Repository in owner/name form" + } + }, + "required": ["gh_auth", "repo"] + }, + "content": "type Github = { token: string }\n\nexport async function main(gh_auth: Github, repo: string) {\n const res = await fetch(`https://api.github.com/repos/${repo}/issues?state=open`, {\n headers: { Authorization: `Bearer ${gh_auth.token}` }\n })\n const issues = await res.json()\n return { repo, open_issues: issues.length }\n}\n" + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/jobs_seed.json b/ai_evals/fixtures/frontend/global/initial/jobs_seed.json index b075d6df26..4c0f923553 100644 --- a/ai_evals/fixtures/frontend/global/initial/jobs_seed.json +++ b/ai_evals/fixtures/frontend/global/initial/jobs_seed.json @@ -15,6 +15,8 @@ "jobKind": "script", "createdBy": "bob", "success": true, + "args": { "region": "emea", "dry_run": false, "recipients": 12 }, + "result": { "sent": 12, "skipped": 3, "digest_url": "https://reports.example.com/d/2026-06-09" }, "logs": "Generating daily digest...\nDigest emailed to 12 recipients\nDone in 1.2s" }, { diff --git a/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json new file mode 100644 index 0000000000..d8cbf37ace --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json @@ -0,0 +1,40 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/notify_customer", + "summary": "Notify a customer", + "description": "Sends a notification to the named customer.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "customer": { + "type": "string" + } + }, + "required": ["customer"] + }, + "value": { + "modules": [ + { + "id": "notify", + "summary": "Send the notification", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(customer: string) {\n return `Notified ${customer}`\n}\n", + "input_transforms": { + "customer": { + "type": "javascript", + "expr": "flow_input.customer" + } + } + } + } + ] + } + } + ] + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/support_chat_flow.json b/ai_evals/fixtures/frontend/global/initial/support_chat_flow.json new file mode 100644 index 0000000000..0db4242ed1 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/support_chat_flow.json @@ -0,0 +1,59 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/support_chat", + "summary": "Support chat", + "description": "Answers customer questions in a chat, remembering earlier messages.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "user_message": { + "type": "string", + "description": "Message from user" + } + }, + "required": ["user_message"] + }, + "value": { + "chat_input_enabled": true, + "modules": [ + { + "id": "assistant", + "summary": "Support assistant", + "value": { + "type": "aiagent", + "tools": [], + "input_transforms": { + "provider": { + "type": "static", + "value": { + "kind": "anthropic", + "model": "claude-haiku-4-5-20251001", + "resource": "$res:f/evals/ai/anthropic" + } + }, + "user_message": { + "type": "javascript", + "expr": "flow_input.user_message" + }, + "system_prompt": { + "type": "static", + "value": "You are a friendly support assistant. Keep answers short." + }, + "memory": { + "type": "static", + "value": { "kind": "auto", "context_length": 10 } + }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + } + } + } + ] + } + } + ] + } +} diff --git a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json b/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json deleted file mode 100644 index abae0ba456..0000000000 --- a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COUNT(*)::bigint AS \"total!\",\n COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS \"replacing!\"\n FROM ai_skill\n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "total!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "replacing!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7" -} diff --git a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json b/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json deleted file mode 100644 index 9c1f8d00a7..0000000000 --- a/backend/.sqlx/query-00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((\n SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM v2_job_queue\n WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1\n )))", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "00e63eab76d26e148b77e932848de74e8b0943d30481465da453942e299a128f" -} diff --git a/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json new file mode 100644 index 0000000000..304efc22d2 --- /dev/null +++ b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n LEFT JOIN password p\n ON p.email = d.email\n AND p.super_admin = true\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)\n ORDER BY d.email NULLS LAST", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username?", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "draft_saved_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f" +} diff --git a/backend/.sqlx/query-0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed.json b/backend/.sqlx/query-0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed.json new file mode 100644 index 0000000000..67f2bbbe81 --- /dev/null +++ b/backend/.sqlx/query-0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COALESCE(dt.value->'database'->>'resource_type', 'unknown') AS \"kind!\",\n COUNT(*)::BIGINT AS \"count!\"\n FROM workspace_settings ws,\n LATERAL jsonb_each(ws.datatable->'datatables') dt\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n GROUP BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "0411a67eb9d88244fa654eda51123e16b5931c08c1073b09ab01dad205f161ed" +} diff --git a/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json new file mode 100644 index 0000000000..be7ac83a7b --- /dev/null +++ b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + false, + false + ] + }, + "hash": "044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55" +} diff --git a/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json new file mode 100644 index 0000000000..6558ea16e5 --- /dev/null +++ b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_shared_artifact\n (workspace_id, artifact_id, email, created_by, name, kind, version, content)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE\n SET created_by = EXCLUDED.created_by,\n name = EXCLUDED.name,\n kind = EXCLUDED.kind,\n version = EXCLUDED.version,\n content = EXCLUDED.content,\n shared_at = now()\n RETURNING id, shared_at, (xmax = 0) AS \"inserted!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "shared_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "inserted!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int4", + "Text" + ] + }, + "nullable": [ + false, + false, + null + ] + }, + "hash": "0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a" +} diff --git a/backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json b/backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json new file mode 100644 index 0000000000..7adefa7eb8 --- /dev/null +++ b/backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3" +} diff --git a/backend/.sqlx/query-06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac.json b/backend/.sqlx/query-06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac.json new file mode 100644 index 0000000000..843207dad7 --- /dev/null +++ b/backend/.sqlx/query-06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n (SELECT MIN(day) FROM guest_activity) AS since,\n (SELECT COUNT(DISTINCT email) FROM guest_activity\n WHERE day > CURRENT_DATE - 30)::INT AS \"guest_count!\",\n (SELECT COUNT(DISTINCT email) FROM guest_activity\n WHERE jwt_entry AND day > CURRENT_DATE - 30)::INT AS \"guest_jwt_count!\",\n (SELECT COUNT(DISTINCT workspace_id) FROM guest_activity\n WHERE day > CURRENT_DATE - 30)::INT AS \"guest_workspace_count!\",\n (SELECT COUNT(*) FROM workspace_settings ws JOIN workspace w ON w.id = ws.workspace_id\n WHERE ws.guest_access_enabled AND NOT w.deleted)::INT AS \"guest_enabled_workspace_count!\",\n (SELECT COUNT(*) FROM workspace WHERE NOT deleted)::INT AS \"workspace_count!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "since", + "type_info": "Date" + }, + { + "ordinal": 1, + "name": "guest_count!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "guest_jwt_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "guest_workspace_count!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "guest_enabled_workspace_count!", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "workspace_count!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null, + null, + null + ] + }, + "hash": "06af616fe4fc61a3b0dcd996a2a1e0e9ac63fc8756aeafac8d279841db117eac" +} diff --git a/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json new file mode 100644 index 0000000000..db9d674fcf --- /dev/null +++ b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',\n 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',\n 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d" +} diff --git a/backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json b/backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json new file mode 100644 index 0000000000..b543f00b44 --- /dev/null +++ b/backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE EXISTS (\n SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d\n WHERE d.value->'reference'->>'workspace_id' = $1\n AND d.value->'reference'->>'datatable' = $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201" +} diff --git a/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json b/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json new file mode 100644 index 0000000000..0aad412ebe --- /dev/null +++ b/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO login_link (token_hash, email, rd, expiration, created_by)\n VALUES ($1, $2, $3, $4, $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bpchar", + "Varchar", + "Text", + "Timestamptz", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02" +} diff --git a/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json b/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json new file mode 100644 index 0000000000..635ec2b94d --- /dev/null +++ b/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a" +} diff --git a/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json b/backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json similarity index 83% rename from backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json rename to backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json index 1c0035f7cb..8aecbed333 100644 --- a/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json +++ b/backend/.sqlx/query-0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary,\n enabled\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -100,8 +105,9 @@ true, false, false, - true + true, + false ] }, - "hash": "6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa" + "hash": "0987f164fe3d64bf0a6a4e9699c2d1f339670da3ed1c08203d54d45798d022dd" } diff --git a/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json b/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json new file mode 100644 index 0000000000..a4d174992e --- /dev/null +++ b/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft AS d\n SET path = $3::text,\n value = (\n SELECT CASE\n WHEN s.clean -> 'path' = to_jsonb($2::text)\n OR s.clean -> 'draft_path' = to_jsonb($2::text)\n THEN to_json(\n s.clean\n || CASE WHEN s.clean -> 'path' = to_jsonb($2::text)\n THEN jsonb_build_object('path', $3::text)\n ELSE '{}'::jsonb END\n || CASE WHEN s.clean -> 'draft_path' = to_jsonb($2::text)\n THEN jsonb_build_object('draft_path', $3::text)\n ELSE '{}'::jsonb END\n )\n ELSE d.value\n END\n FROM (SELECT replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)),\n chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean) s\n )\n WHERE d.workspace_id = $1\n AND d.path = $2::text\n AND d.typ::text = ANY($4::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934" +} diff --git a/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json new file mode 100644 index 0000000000..8275796e90 --- /dev/null +++ b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88" +} diff --git a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json similarity index 78% rename from backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json rename to backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json index f950c5bf57..da25d2fcf8 100644 --- a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json +++ b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236" + "hash": "0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63" } diff --git a/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json new file mode 100644 index 0000000000..446c785770 --- /dev/null +++ b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.email FROM usr u WHERE u.workspace_id = $1 AND u.is_admin AND NOT u.operator AND NOT u.disabled AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) ORDER BY u.username LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388" +} diff --git a/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json b/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json new file mode 100644 index 0000000000..23d3b51c76 --- /dev/null +++ b/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value as \"value!: sqlx::types::Json>\",\n created_at, base\n FROM draft\n WHERE workspace_id = $1\n AND (email = $2 OR email IS NULL)\n AND path = $3\n AND typ = $4\n ORDER BY email NULLS LAST\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "base", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c" +} diff --git a/backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json b/backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json new file mode 100644 index 0000000000..a99ffad770 --- /dev/null +++ b/backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as, permissioned_as_email FROM v2_job\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "permissioned_as_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f" +} diff --git a/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json b/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json new file mode 100644 index 0000000000..93251596cf --- /dev/null +++ b/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft\n SET path = $3,\n -- Both path keys, not just the typed one: the editors mirror the\n -- typed path into the other while it differs from the row's path,\n -- and the loaders prefer the mirror — left naming the old location\n -- it un-does this move on the next save. `create_missing = false`\n -- on both, so a draft carrying only one keeps only one.\n value = to_json(\n jsonb_set(\n jsonb_set(\n CASE WHEN $7::text IS NULL THEN to_jsonb(value)\n ELSE jsonb_set(to_jsonb(value), ARRAY['summary'], to_jsonb($7::text))\n END,\n ARRAY[$5::text], to_jsonb($3::text), false\n ),\n ARRAY[$8::text], to_jsonb($3::text), false\n )\n )\n WHERE workspace_id = $1\n AND path = $2\n AND typ = $4\n AND email = $6\n -- A pre-sanitizer NUL escape makes `to_jsonb` raise 22P05. Excluded\n -- here so the statement can't 500; reported below instead. Unlike the\n -- passive carry, rewriting the value IS this operation, so skipping it\n -- silently would move the row and leave its typed path stale.\n AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0\n -- Skipped on a summary-only edit, where the \"target\" row is this\n -- row and the guard would refuse the update against itself.\n AND ($2 = $3 OR NOT EXISTS (\n SELECT 1 FROM draft o\n WHERE o.workspace_id = $1 AND o.path = $3 AND o.typ::text = ANY($9::text[])\n -- Of this kind only the caller's own row and the legacy one collide:\n -- teammates' drafts of one item share its path by design, but a deploy\n -- there wipes those two together, so a second would discard edits the\n -- caller never saw. The other app kind is a different item on the same\n -- deployed path, so it collides whoever owns it.\n AND (o.typ <> $4 OR o.email = $6 OR o.email IS NULL)\n ))\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1" +} diff --git a/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json new file mode 100644 index 0000000000..42bac71afd --- /dev/null +++ b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM log_file WHERE (hostname, log_ts) IN (\n SELECT hostname, log_ts FROM log_file\n WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval\n LIMIT $2\n ) RETURNING file_path, hostname", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "file_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "hostname", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e" +} diff --git a/backend/.sqlx/query-0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801.json b/backend/.sqlx/query-0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801.json new file mode 100644 index 0000000000..359a7eeefd --- /dev/null +++ b/backend/.sqlx/query-0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = 0, suspend_until = NULL,\n started_at = coalesce(started_at, $2, now())\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "0e277240d2be50383ac53d844c71369067c41870be4561e1c26733ac30419801" +} diff --git a/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json b/backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json similarity index 65% rename from backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json rename to backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json index b82fcf2e17..2ff7820d86 100644 --- a/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json +++ b/backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE NOT $7 OR (datatable_migrations.name = $8::text AND datatable_migrations.code_up = $9::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE datatable_migrations.name = $7::text AND datatable_migrations.code_up = $8::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $9::text", "describe": { "columns": [], "parameters": { @@ -11,7 +11,6 @@ "Varchar", "Text", "Text", - "Bool", "Text", "Text", "Text" @@ -19,5 +18,5 @@ }, "nullable": [] }, - "hash": "2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d" + "hash": "0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3" } diff --git a/backend/.sqlx/query-0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c.json b/backend/.sqlx/query-0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c.json new file mode 100644 index 0000000000..abdea8c942 --- /dev/null +++ b/backend/.sqlx/query-0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO guest_activity (email, workspace_id, day, jwt_entry)\n VALUES ($1, $2, CURRENT_DATE, true)\n ON CONFLICT (email, workspace_id, day)\n DO UPDATE SET jwt_entry = true, last_seen_at = now()\n WHERE NOT guest_activity.jwt_entry\n RETURNING 1 AS \"audited!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "audited!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0fc900f73ef119e4c89186cf120938a298bfe29afe1fd7111340252877f8b86c" +} diff --git a/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json new file mode 100644 index 0000000000..00b839de6a --- /dev/null +++ b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf" +} diff --git a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json deleted file mode 100644 index 3f39982319..0000000000 --- a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6" -} diff --git a/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json b/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json new file mode 100644 index 0000000000..fdf0049344 --- /dev/null +++ b/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108" +} diff --git a/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json b/backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json similarity index 61% rename from backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json rename to backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json index 1818efc0c0..4c120699aa 100644 --- a/backend/.sqlx/query-b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1.json +++ b/backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success)\n VALUES ($1, $2, $3, $4, $5, $6)", + "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", "describe": { "columns": [], "parameters": { @@ -21,10 +21,14 @@ "Text", "Uuid", "Varchar", - "Bool" + "Bool", + "Text", + "Text", + "Text", + "Jsonb" ] }, "nullable": [] }, - "hash": "b1a9a433e577133869c067b2ce383fc6ce4e9df307feb5fd3edc0d1276d61ff1" + "hash": "12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c" } diff --git a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json similarity index 76% rename from backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json rename to backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json index 37115e4ab7..780fb75ea9 100644 --- a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json +++ b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314" + "hash": "139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08" } diff --git a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json b/backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json similarity index 57% rename from backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json rename to backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json index e751b8fc7c..09e5b51ab9 100644 --- a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json +++ b/backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -17,6 +17,11 @@ "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" } ], "parameters": { @@ -28,8 +33,9 @@ "nullable": [ false, true, + false, false ] }, - "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c" + "hash": "148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb" } diff --git a/backend/.sqlx/query-1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1.json b/backend/.sqlx/query-1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1.json new file mode 100644 index 0000000000..3e4b85a916 --- /dev/null +++ b/backend/.sqlx/query-1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token\n (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id)\n VALUES ($1, $2, $3, $4, $5, now() + ($6 || ' seconds')::interval, false, $7, $8)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1553608e0d5a9a9b22c1a2c200bf02200df0007291133033f2b9013c2e508fe1" +} diff --git a/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json new file mode 100644 index 0000000000..03f11ef137 --- /dev/null +++ b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lockfile_hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a" +} diff --git a/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json new file mode 100644 index 0000000000..693d84173a --- /dev/null +++ b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trace_id FROM otel_traces ORDER BY trace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trace_id", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3" +} diff --git a/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json b/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json new file mode 100644 index 0000000000..f5bb8e14d1 --- /dev/null +++ b/backend/.sqlx/query-16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO git_sync_ci_test_check\n (workspace_id, poster_workspace_id, head_sha, head_ref, repo_url,\n repo_resource_path, check_run_id,\n created_at, concluded, conclusion, concluded_at, github_posted)\n VALUES ($1, $2, $3, $4, $5, $6, NULL, now(), false, NULL, NULL, false)\n ON CONFLICT (workspace_id, repo_resource_path, head_sha) DO UPDATE SET\n poster_workspace_id = EXCLUDED.poster_workspace_id,\n head_ref = EXCLUDED.head_ref,\n repo_url = EXCLUDED.repo_url,\n check_run_id = NULL,\n created_at = now(),\n concluded = false,\n conclusion = NULL,\n concluded_at = NULL,\n github_posted = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "16c6e24ae06b52feed597a0c3d299107f989d50ea651546e964670cf99fd2de6" +} diff --git a/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json new file mode 100644 index 0000000000..7bb2cac5db --- /dev/null +++ b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6" +} diff --git a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json b/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json deleted file mode 100644 index bc2e6a6f63..0000000000 --- a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COALESCE(username, split_part(email, '@', 1)) AS \"username!\", email FROM password WHERE super_admin = true AND disabled = false ORDER BY email LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "username!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - false - ] - }, - "hash": "17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3" -} diff --git a/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json new file mode 100644 index 0000000000..73d5000f2d --- /dev/null +++ b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.path = dest.path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef" +} diff --git a/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json b/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json new file mode 100644 index 0000000000..d26cdf3a7d --- /dev/null +++ b/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a\n JOIN LATERAL (\n SELECT av2.id, COALESCE(v.ord, 0) AS ord\n FROM app_version av2\n LEFT JOIN unnest(a.versions) WITH ORDINALITY AS v(id, ord) ON v.id = av2.id\n WHERE av2.app_id = a.id\n ORDER BY ord DESC, av2.id DESC\n LIMIT $3 OFFSET $4\n ) page ON TRUE\n JOIN app_version av ON av.id = page.id\n LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2\n ORDER BY page.ord DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214" +} diff --git a/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json b/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json new file mode 100644 index 0000000000..8647442530 --- /dev/null +++ b/backend/.sqlx/query-18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT test_script_path, tested_item_path, tested_item_kind, has_wildcard AS \"has_wildcard!\" FROM ci_test_reference WHERE workspace_id = $1 ORDER BY test_script_path, tested_item_kind, tested_item_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "test_script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "tested_item_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "tested_item_kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "has_wildcard!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "18ba139acef81d4de18bf21755fa8605c3851b48bab4964c380538ada93f06a9" +} diff --git a/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json new file mode 100644 index 0000000000..f35dbc23e4 --- /dev/null +++ b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM otel_traces WHERE ctid IN (\n SELECT ctid FROM otel_traces\n WHERE start_time_unix_nano < EXTRACT(\n EPOCH FROM now() - ($1::bigint::text || ' s')::interval\n )::bigint * 1000000000\n LIMIT $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e" +} diff --git a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json b/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json similarity index 61% rename from backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json rename to backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json index 0a833ba620..a78e343072 100644 --- a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json +++ b/backend/.sqlx/query-1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT username, email FROM usr WHERE workspace_id = $1 AND is_admin = true AND operator = false AND disabled = false ORDER BY username LIMIT 1", + "query": "SELECT u.username, u.email FROM workspace w JOIN usr u ON u.workspace_id = w.id AND u.email = w.owner WHERE w.id = $1 AND NOT u.disabled", "describe": { "columns": [ { @@ -24,5 +24,5 @@ false ] }, - "hash": "3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c" + "hash": "1b5f6620d35dd74b32ce6325891be02fe144a2da1ebf446b0df92944da791fa7" } diff --git a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json b/backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json similarity index 74% rename from backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json rename to backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json index dcd3573e92..7b78a7d9b4 100644 --- a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json +++ b/backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n SELECT $1, $4, $2, $3, value, now() FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now()\n RETURNING 1 as \"one!\"", + "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value, base\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base)\n SELECT $1, $4, $2, $3, value, now(), base FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now(), base = EXCLUDED.base\n RETURNING 1 as \"one!\"", "describe": { "columns": [ { @@ -55,5 +55,5 @@ null ] }, - "hash": "46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b" + "hash": "1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b" } diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json new file mode 100644 index 0000000000..6b4324af4c --- /dev/null +++ b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3" +} diff --git a/backend/.sqlx/query-209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23.json b/backend/.sqlx/query-209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23.json new file mode 100644 index 0000000000..56128dab99 --- /dev/null +++ b/backend/.sqlx/query-209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' = 'true')::BIGINT AS \"enabled!\",\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' = 'false')::BIGINT AS \"disabled!\",\n COUNT(*) FILTER (WHERE dt.value->>'migrations_enabled' IS NULL)::BIGINT AS \"unset!\"\n FROM workspace_settings ws,\n LATERAL jsonb_each(ws.datatable->'datatables') dt\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "disabled!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "unset!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "209c96d522f9683b39f053707568f9943111e0bac7d4fb478300f0cd8b799f23" +} diff --git a/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json new file mode 100644 index 0000000000..bdb54cfbb9 --- /dev/null +++ b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c" +} diff --git a/backend/.sqlx/query-2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520.json b/backend/.sqlx/query-2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520.json new file mode 100644 index 0000000000..c1ff273bcb --- /dev/null +++ b/backend/.sqlx/query-2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*)::BIGINT AS \"total!\",\n COUNT(DISTINCT (workspace_id, datatable))::BIGINT AS \"datatables!\"\n FROM datatable_migrations", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "datatables!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "2105d37be923a445933c899bb31523709f837b7e1969b7364f70cc2a60cdd520" +} diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json new file mode 100644 index 0000000000..04b23832d9 --- /dev/null +++ b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" +} diff --git a/backend/.sqlx/query-21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe.json b/backend/.sqlx/query-21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe.json new file mode 100644 index 0000000000..ca9f2cbc29 --- /dev/null +++ b/backend/.sqlx/query-21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO guest_activity (email, workspace_id, day)\n VALUES ($1, $2, CURRENT_DATE)\n ON CONFLICT (email, workspace_id, day)\n DO UPDATE SET last_seen_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "21e9629bdf5824b676bf88709f4fe0d9644b8d6a08d0c73daac73c48b5933afe" +} diff --git a/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json new file mode 100644 index 0000000000..50755132fc --- /dev/null +++ b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4,\n 'source', $4, $5, '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6,\n 'model', $6, $7, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c" +} diff --git a/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json b/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json new file mode 100644 index 0000000000..f32e47b387 --- /dev/null +++ b/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email)\n SELECT m.workspace_id, m.typ, m.old_path, $4, $5::text\n FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ::text = ANY($2::text[])\n AND m.new_path = $3 AND m.email IS NULL\n AND NOT EXISTS (\n SELECT 1 FROM draft_move o\n WHERE o.workspace_id = m.workspace_id AND o.typ = m.typ\n AND o.old_path = m.old_path AND o.email = $5::text\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48" +} diff --git a/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json new file mode 100644 index 0000000000..4ae74ff53c --- /dev/null +++ b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT cost_nanos FROM ai_free_token_usage WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5" +} diff --git a/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json b/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json new file mode 100644 index 0000000000..29ec70e75e --- /dev/null +++ b/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330" +} diff --git a/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json new file mode 100644 index 0000000000..5f3d8300eb --- /dev/null +++ b/backend/.sqlx/query-2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH slots AS (\n SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak,\n (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value,\n (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing,\n COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp,\n max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb\n FROM (\n SELECT id, t,\n CASE jsonb_typeof(value)\n WHEN 'number' THEN value::double precision\n WHEN 'object' THEN t - (value->>'since')::double precision\n END AS v,\n (value->>'since')::double precision AS since,\n (jsonb_typeof(value) = 'object')::int::double precision AS climbing,\n greatest(floor((t - $1::double precision) / $2::double precision), -1)::int\n AS slot\n FROM (\n SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > to_timestamp($1::double precision - $3::double precision)\n ) m\n ) s\n WHERE v IS NOT NULL\n GROUP BY id, slot\n )\n SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\",\n greatest(peak, CASE WHEN last_climb[1] < last THEN (\n SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision\n FROM metrics n\n WHERE n.id = slots.id AND n.id LIKE 'queue_%'\n AND n.created_at > to_timestamp(last_climb[1] + 0.001)\n AND n.created_at <= to_timestamp(last + 0.001)\n ) - last_climb[2] END) AS \"peak!\",\n first_value AS \"first_value!\", last_value AS \"last_value!\",\n last_climbing AS \"last_climbing!\", ramp AS \"ramp!\"\n FROM slots\n ORDER BY id, slot", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slot!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "first!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "last!", + "type_info": "Float8" + }, + { + "ordinal": 4, + "name": "peak!", + "type_info": "Float8" + }, + { + "ordinal": 5, + "name": "first_value!", + "type_info": "Float8" + }, + { + "ordinal": 6, + "name": "last_value!", + "type_info": "Float8" + }, + { + "ordinal": 7, + "name": "last_climbing!", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "ramp!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Float8", + "Float8", + "Float8", + "Float8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "2742245bc03290120a97b21c441cb56825e9fd552a7aeddfb8a372540c19b863" +} diff --git a/backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json b/backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json new file mode 100644 index 0000000000..cda61b894a --- /dev/null +++ b/backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $2\n THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98" +} diff --git a/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json b/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json deleted file mode 100644 index e4f70e1f07..0000000000 --- a/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_completed SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3" -} diff --git a/backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json b/backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json new file mode 100644 index 0000000000..a893edfbd7 --- /dev/null +++ b/backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213" +} diff --git a/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json new file mode 100644 index 0000000000..919d930f70 --- /dev/null +++ b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887" +} diff --git a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json b/backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json similarity index 65% rename from backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json rename to backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json index 697e49ab9d..3a864fee63 100644 --- a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json +++ b/backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json @@ -1,15 +1,23 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb\n RETURNING duration_ms AS \"duration_ms!\"", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "duration_ms!", + "type_info": "Int8" + } + ], "parameters": { "Left": [ "Uuid", "Jsonb" ] }, - "nullable": [] + "nullable": [ + false + ] }, - "hash": "beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb" + "hash": "2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f" } diff --git a/backend/.sqlx/query-2d6b117324eaf076a0ed06d2cb0ce73279957d6a39fdc6b1ecb6e0a1e02f921f.json b/backend/.sqlx/query-2d6b117324eaf076a0ed06d2cb0ce73279957d6a39fdc6b1ecb6e0a1e02f921f.json new file mode 100644 index 0000000000..4b96519a06 --- /dev/null +++ b/backend/.sqlx/query-2d6b117324eaf076a0ed06d2cb0ce73279957d6a39fdc6b1ecb6e0a1e02f921f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)\n SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account,\n CASE WHEN $3 THEN NULL ELSE added_via END\n FROM usr WHERE workspace_id = $2\n AND (NOT $3 OR (NOT operator AND NOT disabled AND NOT is_service_account))\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "2d6b117324eaf076a0ed06d2cb0ce73279957d6a39fdc6b1ecb6e0a1e02f921f" +} diff --git a/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json new file mode 100644 index 0000000000..7148252002 --- /dev/null +++ b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188" +} diff --git a/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json b/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json new file mode 100644 index 0000000000..99c8262860 --- /dev/null +++ b/backend/.sqlx/query-2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n route_path = $1,\n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n allowed_origins = $6,\n authentication_resource_path = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n mode = $11,\n http_method = $12,\n static_asset_config = $13,\n edited_by = $14,\n permissioned_as = $15,\n request_type = $16,\n authentication_method = $17,\n summary = $18,\n description = $19,\n edited_at = now(),\n is_static_website = $20,\n error_handler_path = $21,\n error_handler_args = $22,\n retry = $23\n WHERE\n workspace_id = $24 AND\n path = $25\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2ead4c5e0fec64dfdc24431d2f4871ca8667a657dfdfc2fa65e9ff1a3c0d2908" +} diff --git a/backend/.sqlx/query-2f39fce0ee700117f3e4c066e0b56ee979e4c392970278304e80368b770bb7b4.json b/backend/.sqlx/query-2f39fce0ee700117f3e4c066e0b56ee979e4c392970278304e80368b770bb7b4.json new file mode 100644 index 0000000000..99b1a9472d --- /dev/null +++ b/backend/.sqlx/query-2f39fce0ee700117f3e4c066e0b56ee979e4c392970278304e80368b770bb7b4.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "add_admins_and_developers_to_forks", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2f39fce0ee700117f3e4c066e0b56ee979e4c392970278304e80368b770bb7b4" +} diff --git a/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json b/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json new file mode 100644 index 0000000000..54da0105e7 --- /dev/null +++ b/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE password SET password_hash = $1, login_type = 'password'\n WHERE email = $2 AND login_type = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d" +} diff --git a/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json new file mode 100644 index 0000000000..0937a312d1 --- /dev/null +++ b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094" +} diff --git a/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json b/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json new file mode 100644 index 0000000000..941dc122d0 --- /dev/null +++ b/backend/.sqlx/query-32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "32b4d1fe69fd219a5931fdd63cb2b0cb7770950e5c7fa6128bc9d2b2abcc7f2f" +} diff --git a/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json new file mode 100644 index 0000000000..0a2976f868 --- /dev/null +++ b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "flow_step_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9" +} diff --git a/backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json b/backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json new file mode 100644 index 0000000000..0c94309c58 --- /dev/null +++ b/backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + null + ] + }, + "hash": "334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json b/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json new file mode 100644 index 0000000000..9f9cf036fb --- /dev/null +++ b/backend/.sqlx/query-35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->>'script_path' as \"script_path\"\n FROM workspace_settings ws,\n jsonb_array_elements(\n CASE WHEN jsonb_typeof(ws.git_sync->'repositories') = 'array'\n THEN ws.git_sync->'repositories' END\n ) r\n WHERE ws.workspace_id = $1\n AND r->>'git_repo_resource_path' IN ($2, '$res:' || $2)\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "35783f52031d7ba14142108480b3599f083dee7415acdbe4410412309dfa2ca1" +} diff --git a/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json b/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json deleted file mode 100644 index 23cf739f90..0000000000 --- a/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH all_audit AS (SELECT username, operation, timestamp FROM audit_partitioned UNION ALL SELECT username, operation, timestamp FROM audit),\n active_users as (SELECT distinct username as email FROM all_audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors", - "type_info": "VarcharArray" - }, - { - "ordinal": 1, - "name": "operators", - "type_info": "VarcharArray" - }, - { - "ordinal": 2, - "name": "author_count", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "operator_count", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b" -} diff --git a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json similarity index 76% rename from backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json rename to backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json index d2084d76d8..76e897d3a4 100644 --- a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json +++ b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -58,8 +63,9 @@ true, true, true, - false + false, + true ] }, - "hash": "623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4" + "hash": "36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907" } diff --git a/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json b/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json new file mode 100644 index 0000000000..ebaaf0b755 --- /dev/null +++ b/backend/.sqlx/query-3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of_email' = $2 AND (value->'policy'->>'on_behalf_of' IS NULL OR value->'policy'->>'on_behalf_of' NOT LIKE 'g/%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3721bd6524ea48a1068ee8013bcc1aeca1b9fe784336fabb71ce13bdb58839da" +} diff --git a/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json b/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json new file mode 100644 index 0000000000..8a780ccda7 --- /dev/null +++ b/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n AND NOT starts_with(label, 'cli-login:')\n ))\n RETURNING token_prefix", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99" +} diff --git a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json similarity index 55% rename from backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json rename to backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json index d453eb430d..2f794e6be0 100644 --- a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json +++ b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", "describe": { "columns": [ { @@ -13,12 +13,16 @@ "Left": [ "Text", "Jsonb", - "Text" + "Text", + "Text", + "Bool", + "Text", + "Bool" ] }, "nullable": [ null ] }, - "hash": "1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae" + "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" } diff --git a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json similarity index 75% rename from backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json rename to backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json index e4db87ec7d..87d740a247 100644 --- a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json +++ b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545" + "hash": "38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a" } diff --git a/backend/.sqlx/query-391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1.json b/backend/.sqlx/query-391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1.json new file mode 100644 index 0000000000..8fcb8140f4 --- /dev/null +++ b/backend/.sqlx/query-391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM app WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "391139a04bd48319a5512e7859b63e81438c7483ac892b971fe8a20709555cc1" +} diff --git a/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json b/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json new file mode 100644 index 0000000000..c4cc310dbf --- /dev/null +++ b/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email)\n SELECT $1, t::draft_kind, $3, $4, $5 FROM unnest($2::text[]) t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c" +} diff --git a/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json new file mode 100644 index 0000000000..28b34afa97 --- /dev/null +++ b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,\n ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,\n child_unique_id, child_column, lineage_kind, ingested_at\n FROM dbt_column_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4" +} diff --git a/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json new file mode 100644 index 0000000000..843484dd86 --- /dev/null +++ b/backend/.sqlx/query-3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "now!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "3bd816e986ef2d2a193e51c985b61c04b71f464021e4b884bf21cc7c25f6a753" +} diff --git a/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json b/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json new file mode 100644 index 0000000000..db2181ae3c --- /dev/null +++ b/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET new_path = $4\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND new_path = $3 AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09" +} diff --git a/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json new file mode 100644 index 0000000000..1268656e34 --- /dev/null +++ b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest, manifest_key, run_results,\n run_results_key, updated_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET\n job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest,\n manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results,\n run_results_key = EXCLUDED.run_results_key, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Uuid", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3" +} diff --git a/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json new file mode 100644 index 0000000000..564c784d22 --- /dev/null +++ b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin AS \"super_admin!\" FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c" +} diff --git a/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json b/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json new file mode 100644 index 0000000000..2d0bb40c18 --- /dev/null +++ b/backend/.sqlx/query-422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET concluded = true, github_posted = true, concluded_at = now(),\n conclusion = COALESCE(conclusion, 'failure')\n WHERE (check_run_id IS NULL AND NOT concluded\n AND created_at < now() - make_interval(secs => $1))\n OR (concluded AND NOT github_posted\n AND concluded_at < now() - make_interval(secs => $2))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "422490f2f91b4d97331e87da135884932eab27f53171c12291cca15a6ec33586" +} diff --git a/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json b/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json new file mode 100644 index 0000000000..5522f3bb17 --- /dev/null +++ b/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT profile FROM cloud_onboarding_profile WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "profile", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743" +} diff --git a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json similarity index 50% rename from backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json rename to backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json index 124a6e36c5..f7c39c21fc 100644 --- a/backend/.sqlx/query-8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15.json +++ b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json @@ -1,15 +1,15 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value) VALUES ($1, $2)", + "query": "UPDATE draft SET email = $1 WHERE email = $2", "describe": { "columns": [], "parameters": { "Left": [ "Varchar", - "Jsonb" + "Text" ] }, "nullable": [] }, - "hash": "8824b382c4e98dfa17b4aa656af3a6c1ff99973e778d71bd598a50d022da8f15" + "hash": "43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8" } diff --git a/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json new file mode 100644 index 0000000000..7f950346ee --- /dev/null +++ b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4" +} diff --git a/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json new file mode 100644 index 0000000000..2253944269 --- /dev/null +++ b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, $1::bigint, now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = ai_free_token_daily_usage.cost_nanos + $1::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [ + false + ] + }, + "hash": "44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996" +} diff --git a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json b/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json deleted file mode 100644 index 9f97b97a88..0000000000 --- a/backend/.sqlx/query-44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH queue_metrics as (\n SELECT id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "values!", - "type_info": "JsonArray" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null - ] - }, - "hash": "44dd7a66ecc9564ad5727970b5f60a1717eda8924999741be522ffb74cc173fa" -} diff --git a/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json b/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json new file mode 100644 index 0000000000..526f3f5537 --- /dev/null +++ b/backend/.sqlx/query-45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "45d0e716fa402a63b0bf6877c21c0fa50b81d46845c400d569d6d8e49e503b59" +} diff --git a/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json b/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json deleted file mode 100644 index 0c4d90b073..0000000000 --- a/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c" -} diff --git a/backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json b/backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json new file mode 100644 index 0000000000..b0ad1eaf86 --- /dev/null +++ b/backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601" +} diff --git a/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json new file mode 100644 index 0000000000..9b361b8f53 --- /dev/null +++ b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru\n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf" +} diff --git a/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json b/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json new file mode 100644 index 0000000000..6bec80ccb2 --- /dev/null +++ b/backend/.sqlx/query-47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n http_trigger\n SET\n wrap_body = $1,\n raw_string = $2,\n allowed_origins = $3,\n authentication_resource_path = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n mode = $8,\n http_method = $9,\n static_asset_config = $10,\n edited_by = $11,\n permissioned_as = $12,\n request_type = $13,\n authentication_method = $14,\n summary = $15,\n description = $16,\n edited_at = now(),\n is_static_website = $17,\n error_handler_path = $18,\n error_handler_args = $19,\n retry = $20\n WHERE\n workspace_id = $21 AND\n path = $22\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Varchar", + "Jsonb", + "Jsonb", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "47e0f46fddb3ad1c854deb9bdbdcbc2bc7235c63ed4f0a885d412793f3a3a3fc" +} diff --git a/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json new file mode 100644 index 0000000000..b2cfb1cd89 --- /dev/null +++ b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem->'credential'\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') AS elem\n WHERE workspace_id = $1 AND elem->>'git_repo_resource_path' IN ($2, $3)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6" +} diff --git a/backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json b/backend/.sqlx/query-48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31.json similarity index 77% rename from backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json rename to backend/.sqlx/query-48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31.json index 56a3642faa..1b50ef4134 100644 --- a/backend/.sqlx/query-c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9.json +++ b/backend/.sqlx/query-48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2", + "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "created_by", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_test", + "type_info": "Bool" } ], "parameters": { @@ -52,8 +57,9 @@ true, false, false, + false, false ] }, - "hash": "c383cc023714b361d10c10e8fef1fc148ab1da942951ee9ffdddaecee76a6be9" + "hash": "48c8522a4fed219c5011f4ba63c81cfe028a8b2a32bd790840cef65c452a8c31" } diff --git a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json b/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json deleted file mode 100644 index 8d86ff3db6..0000000000 --- a/backend/.sqlx/query-49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Text", - "Varchar", - "Int8", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "49b18e987e2dfa3c7ab915757ff3b9c0e6e371136b565f9b0f5a3393ef8d8d57" -} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json b/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json deleted file mode 100644 index 63283a830a..0000000000 --- a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at)\n VALUES ('admins', $1, $2, $3, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb" -} diff --git a/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json new file mode 100644 index 0000000000..681206a381 --- /dev/null +++ b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668" +} diff --git a/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json new file mode 100644 index 0000000000..913c16c31d --- /dev/null +++ b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt'\n AND (hash = $4 OR $4 = ANY(parent_hashes)))\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0" +} diff --git a/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json b/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json new file mode 100644 index 0000000000..8c784b664e --- /dev/null +++ b/backend/.sqlx/query-4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = now()\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4\n AND (tests_dispatched_at IS NULL\n OR (ci_test_job_ids IS NULL\n AND tests_dispatched_at < now() - make_interval(secs => $5)))\n RETURNING true as \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4dde939e92f5b8a9cc165c9ea383a456eef081d16c2a58e7262f86d80289c2da" +} diff --git a/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json b/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json new file mode 100644 index 0000000000..c8b8b94da4 --- /dev/null +++ b/backend/.sqlx/query-4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (\n SELECT h.sha FROM git_sync_synced_head h\n WHERE h.workspace_id = $1 AND h.repo_resource_path = $4 AND h.branch = $3\n ORDER BY h.synced_at DESC LIMIT 1\n ) = $2 AND NOT EXISTS (\n SELECT 1\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.kind = 'deploymentcallback'\n AND j.args->'__git_sync_auto_pull'->>'branch' = $3\n AND j.args->'__git_sync_auto_pull'->>'repo_resource_path'\n IN ($4, '$res:' || $4)\n ) AND NOT EXISTS (\n SELECT 1\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.kind IN ('dependencies', 'flowdependencies', 'appdependencies')\n ) as \"ready\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ready", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4e4b31e97f0cc946f26cc0faf9a09de2846c43ee48f047e0f2ea9ee7a6502c81" +} diff --git a/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json b/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json new file mode 100644 index 0000000000..f7331a55e8 --- /dev/null +++ b/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company, username, first_time_user)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c" +} diff --git a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json similarity index 70% rename from backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json rename to backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json index da77dc1de5..134e58ed58 100644 --- a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json +++ b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.column_schema, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", "describe": { "columns": [ { @@ -80,21 +80,26 @@ }, { "ordinal": 15, - "name": "freshness", + "name": "column_schema", "type_info": "Jsonb" }, { "ordinal": 16, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 17, "name": "raw_code", "type_info": "Text" }, { - "ordinal": 17, + "ordinal": 18, "name": "original_file_path", "type_info": "Text" }, { - "ordinal": 18, + "ordinal": 19, "name": "script_visible!", "type_info": "Bool" } @@ -127,8 +132,9 @@ true, true, true, + true, null ] }, - "hash": "9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12" + "hash": "4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5" } diff --git a/backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json b/backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json new file mode 100644 index 0000000000..a5695f75fd --- /dev/null +++ b/backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM flow_conversation WHERE id = ANY($1) AND workspace_id = $2 ORDER BY id FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef" +} diff --git a/backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json b/backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json new file mode 100644 index 0000000000..804e294296 --- /dev/null +++ b/backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57" +} diff --git a/backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json b/backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json new file mode 100644 index 0000000000..03613519ad --- /dev/null +++ b/backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2" +} diff --git a/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json new file mode 100644 index 0000000000..21759e2fe5 --- /dev/null +++ b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT p.asset_path AS \"from_path!\", e.parent_column AS \"from_column!\",\n c.asset_path AS \"to_path!\", e.child_column AS \"to_column!\",\n e.lineage_kind AS \"kind!\"\n FROM unnest($2::text[], $3::bigint[], $4::uuid[])\n AS o(script_path, script_hash, job_id)\n JOIN dbt_column_edge e ON e.workspace_id = $1\n AND e.script_path = o.script_path\n AND e.job_id = o.job_id\n -- `=` still, with the NULL-to-NULL case\n -- spelled out and gated on the pin: a\n -- version-less row's hash is NULL on both\n -- sides, which `=` never matches, but\n -- `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound everywhere else.\n AND (e.script_hash = o.script_hash\n OR ($5::text IS NOT NULL\n AND o.script_hash IS NULL\n AND e.script_hash IS NULL))\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND p.script_hash IS NOT DISTINCT FROM e.script_hash\n AND p.job_id = e.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND c.script_hash IS NOT DISTINCT FROM e.script_hash\n AND c.job_id = e.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.lineage_kind IN ('copy', 'mod')\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "from_column!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "to_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "to_column!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array", + "UuidArray", + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + false + ] + }, + "hash": "55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d" +} diff --git a/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json new file mode 100644 index 0000000000..822695a6c9 --- /dev/null +++ b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, $2, $2, '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d" +} diff --git a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json b/backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json similarity index 56% rename from backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json rename to backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json index 29ce0c6f2c..f4f7acd0b7 100644 --- a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json +++ b/backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", + "query": "SELECT username FROM usr WHERE workspace_id = $1 AND email = $2 AND disabled = false", "describe": { "columns": [ { "ordinal": 0, - "name": "name", + "name": "username", "type_info": "Varchar" } ], @@ -19,5 +19,5 @@ false ] }, - "hash": "e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61" + "hash": "58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018" } diff --git a/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json new file mode 100644 index 0000000000..b2a0d728cd --- /dev/null +++ b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false AND language = 'dbt'\n AND (hash = $3 OR $3 = ANY(parent_hashes))\n FOR SHARE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147" +} diff --git a/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json new file mode 100644 index 0000000000..f0d9aabea3 --- /dev/null +++ b/backend/.sqlx/query-58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\",\n EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\",\n EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\"\n FROM unnest($1::text[]) AS c(id)\n FULL JOIN (\n SELECT DISTINCT ON (id) id, value, created_at\n FROM metrics\n WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2)\n ORDER BY id, created_at DESC\n ) r ON r.id = c.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "value?", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "at?", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "age?", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Float8" + ] + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "58f99e0d5877f403cde04459e4a425efd77aa4c0118f62ada2caa0794b0d738b" +} diff --git a/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json b/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json new file mode 100644 index 0000000000..b78808a389 --- /dev/null +++ b/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft_move src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.old_path = dest.old_path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f" +} diff --git a/backend/.sqlx/query-5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658.json b/backend/.sqlx/query-5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658.json new file mode 100644 index 0000000000..8864b75b79 --- /dev/null +++ b/backend/.sqlx/query-5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE flow_conversation SET title = $1, updated_at = updated_at\n WHERE id = $2 AND workspace_id = $3\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5b9c9eb64051f291fed4be9bc0b0cc0aef2e7bde732899976eddac36a2da7658" +} diff --git a/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json b/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json new file mode 100644 index 0000000000..08008e5900 --- /dev/null +++ b/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin, devops FROM password WHERE email = $1 AND disabled = false FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "devops", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f" +} diff --git a/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json new file mode 100644 index 0000000000..09efdc7340 --- /dev/null +++ b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521" +} diff --git a/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json b/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json new file mode 100644 index 0000000000..c1d11bffe1 --- /dev/null +++ b/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM cloud_trial_offer WHERE email = $1 AND consumed_at IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060" +} diff --git a/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json new file mode 100644 index 0000000000..8b04c9e258 --- /dev/null +++ b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10" +} diff --git a/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json b/backend/.sqlx/query-5ccfbd0f345b9b86ca356008def6cc1011a49fc8d8ad046ac572e81cf8938995.json similarity index 63% rename from backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json rename to backend/.sqlx/query-5ccfbd0f345b9b86ca356008def6cc1011a49fc8d8ad046ac572e81cf8938995.json index 0532a5d3a0..7cd917d623 100644 --- a/backend/.sqlx/query-a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a.json +++ b/backend/.sqlx/query-5ccfbd0f345b9b86ca356008def6cc1011a49fc8d8ad046ac572e81cf8938995.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_settings\n SET\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n dbt_warehouses = source_ws.dbt_warehouses,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", + "query": "\n UPDATE workspace_settings\n SET\n ai_config = source_ws.ai_config,\n large_file_storage = source_ws.large_file_storage,\n ducklake = source_ws.ducklake,\n dbt_warehouses = source_ws.dbt_warehouses,\n datatable = source_ws.datatable,\n git_app_installations = source_ws.git_app_installations,\n add_admins_and_developers_to_forks = source_ws.add_admins_and_developers_to_forks\n FROM workspace_settings source_ws\n WHERE source_ws.workspace_id = $1\n AND workspace_settings.workspace_id = $2\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "a6decdebcd9750691f20f874b66a9c6f2ede08c33605c6a3c3dfc213a3eda76a" + "hash": "5ccfbd0f345b9b86ca356008def6cc1011a49fc8d8ad046ac572e81cf8938995" } diff --git a/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json b/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json new file mode 100644 index 0000000000..53a3b04d6b --- /dev/null +++ b/backend/.sqlx/query-5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET ci_test_job_ids = NULL, tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5fcaf17e24fa00ffafdc5f0f425fe4c1c745457d463d8ea61627ebe61ba8ab2c" +} diff --git a/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json b/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json new file mode 100644 index 0000000000..f50b4b743f --- /dev/null +++ b/backend/.sqlx/query-61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific\n WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "61caaa5c4ae62f618ba18f36330bbf58ff58e8c448874ffc9ca466165e545878" +} diff --git a/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json new file mode 100644 index 0000000000..3713af1486 --- /dev/null +++ b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::text[]) AS c(hostname, file_path)\n WHERE log_file.indexed_at IS NULL\n AND log_file.hostname = c.hostname\n AND log_file.file_path = c.file_path", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400" +} diff --git a/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json new file mode 100644 index 0000000000..67b672228f --- /dev/null +++ b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano)\n VALUES ($1, $2, 'GET /', 3, $3, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc" +} diff --git a/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json b/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json new file mode 100644 index 0000000000..1eb8c87315 --- /dev/null +++ b/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO cloud_onboarding_profile (email, profile, created_by) VALUES ($1, $2, $3)\n ON CONFLICT (email) DO UPDATE SET profile = EXCLUDED.profile", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0" +} diff --git a/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json b/backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json similarity index 63% rename from backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json rename to backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json index 89a02933e0..d234750140 100644 --- a/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json +++ b/backend/.sqlx/query-64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", + "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary,\n enabled\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", "describe": { "columns": [], "parameters": { @@ -23,10 +23,11 @@ "Bool", "Varchar", "Jsonb", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e" + "hash": "64dce306efc0d9989542dba5bf003dc94b9337a8d3b69805308258d1fa145e63" } diff --git a/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json new file mode 100644 index 0000000000..dc1f6ffbb1 --- /dev/null +++ b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2" +} diff --git a/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json new file mode 100644 index 0000000000..8dd7954460 --- /dev/null +++ b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb" +} diff --git a/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json new file mode 100644 index 0000000000..4481fa3dea --- /dev/null +++ b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT max(log_ts) FROM log_file\n WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json b/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json new file mode 100644 index 0000000000..52105ca605 --- /dev/null +++ b/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)\n RETURNING m.conversation_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "conversation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976" +} diff --git a/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json new file mode 100644 index 0000000000..b1b92ecc75 --- /dev/null +++ b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "resource_type", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0" +} diff --git a/backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json b/backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json index 996390d8ca..46f8d77060 100644 --- a/backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json +++ b/backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json @@ -1,6 +1,10 @@ { "db_name": "PostgreSQL", +<<<<<<<< HEAD:backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, variable_expiration_handler) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, variable_expiration_handler FROM workspace_settings WHERE workspace_id = $2", +======== + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", +>>>>>>>> origin/main:backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json "describe": { "columns": [], "parameters": { @@ -11,5 +15,9 @@ }, "nullable": [] }, +<<<<<<<< HEAD:backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json "hash": "6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da" +======== + "hash": "77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1" +>>>>>>>> origin/main:backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json } diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json similarity index 86% rename from backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json rename to backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json index ba9f3814e8..13d1c9b1e8 100644 --- a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json +++ b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > $1\n ORDER BY log_ts ASC LIMIT $2", + "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE indexed_at IS NULL\n ORDER BY log_ts ASC, hostname ASC LIMIT $1", "describe": { "columns": [ { @@ -46,7 +46,6 @@ ], "parameters": { "Left": [ - "Timestamp", "Int8" ] }, @@ -61,5 +60,5 @@ true ] }, - "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a" + "hash": "6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1" } diff --git a/backend/.sqlx/query-94fd0a57cfc9341b2e9deae60506c6c06aa6934b87200da14231f12f65149cd3.json b/backend/.sqlx/query-6c57c46c5a0462f379ed6a22ae97e363009177fdc45d795447a920f455f193cc.json similarity index 77% rename from backend/.sqlx/query-94fd0a57cfc9341b2e9deae60506c6c06aa6934b87200da14231f12f65149cd3.json rename to backend/.sqlx/query-6c57c46c5a0462f379ed6a22ae97e363009177fdc45d795447a920f455f193cc.json index d54260b4f7..b8a19247ca 100644 --- a/backend/.sqlx/query-94fd0a57cfc9341b2e9deae60506c6c06aa6934b87200da14231f12f65149cd3.json +++ b/backend/.sqlx/query-6c57c46c5a0462f379ed6a22ae97e363009177fdc45d795447a920f455f193cc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM token_expiry_notification n\n USING token t\n WHERE n.token_hash = t.token_hash\n AND n.expiration > now()\n AND n.expiration <= now() + interval '7 days'\n RETURNING t.token_prefix, t.label, t.email, t.workspace_id", + "query": "DELETE FROM token_expiry_notification n\n USING token t\n WHERE n.token_hash = t.token_hash\n AND n.expiration > now()\n AND n.expiration <= now() + make_interval(days => $1)\n RETURNING t.token_prefix, t.label, t.email, t.workspace_id", "describe": { "columns": [ { @@ -25,7 +25,9 @@ } ], "parameters": { - "Left": [] + "Left": [ + "Int4" + ] }, "nullable": [ false, @@ -34,5 +36,5 @@ true ] }, - "hash": "94fd0a57cfc9341b2e9deae60506c6c06aa6934b87200da14231f12f65149cd3" + "hash": "6c57c46c5a0462f379ed6a22ae97e363009177fdc45d795447a920f455f193cc" } diff --git a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json b/backend/.sqlx/query-6d259b8cce5da5fecefe4ce322789b6b2cc43b51f2056c677d58f39c31fb26cb.json similarity index 71% rename from backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json rename to backend/.sqlx/query-6d259b8cce5da5fecefe4ce322789b6b2cc43b51f2056c677d58f39c31fb26cb.json index d7f5fc45d4..b666243d40 100644 --- a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json +++ b/backend/.sqlx/query-6d259b8cce5da5fecefe4ce322789b6b2cc43b51f2056c677d58f39c31fb26cb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title, is_test)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (id) DO NOTHING\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "created_by", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_test", + "type_info": "Bool" } ], "parameters": { @@ -45,7 +50,8 @@ "Varchar", "Varchar", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [ @@ -55,8 +61,9 @@ true, false, false, + false, false ] }, - "hash": "6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe" + "hash": "6d259b8cce5da5fecefe4ce322789b6b2cc43b51f2056c677d58f39c31fb26cb" } diff --git a/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json new file mode 100644 index 0000000000..8026b07416 --- /dev/null +++ b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->'auto_pull'->>'enabled_by' AS \"enabled_by\"\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') r\n WHERE workspace_id = $1 AND r->>'git_repo_resource_path' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165" +} diff --git a/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json b/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json new file mode 100644 index 0000000000..427ba00075 --- /dev/null +++ b/backend/.sqlx/query-6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET ci_test_job_ids = $5\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "6e3cd83ad7eef0dddacf9359f662193e094ca44642342778fb4281a711263385" +} diff --git a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json similarity index 78% rename from backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json rename to backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json index c44d3d711d..e444c0d549 100644 --- a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json +++ b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -55,8 +60,9 @@ true, true, true, - false + false, + true ] }, - "hash": "e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3" + "hash": "6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251" } diff --git a/backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json b/backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json new file mode 100644 index 0000000000..d87d3aeba5 --- /dev/null +++ b/backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909" +} diff --git a/backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json b/backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json deleted file mode 100644 index 17fbb0d5bb..0000000000 --- a/backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH capped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n UNION ALL\n SELECT timestamp, operation, resource, parameters\n FROM audit\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n ORDER BY timestamp DESC\n LIMIT 200\n ), uncapped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n UNION ALL\n SELECT timestamp, operation, resource, parameters\n FROM audit\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n ORDER BY timestamp DESC\n LIMIT 200\n )\n SELECT timestamp AS \"timestamp!\",\n operation::text AS \"operation!\",\n resource AS workspace_id,\n parameters\n FROM (SELECT * FROM capped UNION ALL SELECT * FROM uncapped) e\n ORDER BY timestamp DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "timestamp!", - "type_info": "Timestamptz" - }, - { - "ordinal": 1, - "name": "operation!", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "parameters", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a" -} diff --git a/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json b/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json new file mode 100644 index 0000000000..a03fab7081 --- /dev/null +++ b/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, enabled, pwd FROM datatable_role", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "pwd", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0" +} diff --git a/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json new file mode 100644 index 0000000000..3a67641732 --- /dev/null +++ b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 6000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837" +} diff --git a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json b/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json deleted file mode 100644 index d55722ed24..0000000000 --- a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by)\n VALUES ($1, $2, $3, $4, now(), $5)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET description = EXCLUDED.description,\n instructions = EXCLUDED.instructions,\n edited_at = now(),\n edited_by = EXCLUDED.edited_by", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8" -} diff --git a/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json new file mode 100644 index 0000000000..7c11dc80c0 --- /dev/null +++ b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232" +} diff --git a/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json b/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json new file mode 100644 index 0000000000..3136e1b433 --- /dev/null +++ b/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE login_link SET consumed_at = now()\n WHERE token_hash = $1 AND consumed_at IS NULL AND expiration > now()\n RETURNING email, rd, created_by", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "rd", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c" +} diff --git a/backend/.sqlx/query-75e6b5cd52d63ac094c90abd1524bc7a919692a96b076b8aaf24d812723531c8.json b/backend/.sqlx/query-75e6b5cd52d63ac094c90abd1524bc7a919692a96b076b8aaf24d812723531c8.json new file mode 100644 index 0000000000..27c62583f1 --- /dev/null +++ b/backend/.sqlx/query-75e6b5cd52d63ac094c90abd1524bc7a919692a96b076b8aaf24d812723531c8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM usr WHERE email = $1 AND is_service_account IS true\n AND ($2::varchar IS NULL OR workspace_id = $2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "75e6b5cd52d63ac094c90abd1524bc7a919692a96b076b8aaf24d812723531c8" +} diff --git a/backend/.sqlx/query-76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477.json b/backend/.sqlx/query-76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477.json new file mode 100644 index 0000000000..ebd492abf3 --- /dev/null +++ b/backend/.sqlx/query-76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET guest_access_enabled = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "76c0331b18eed478a50572e35642909be7dc7eb9b6deac7ea439eb637d728477" +} diff --git a/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json new file mode 100644 index 0000000000..8eac288a1b --- /dev/null +++ b/backend/.sqlx/query-7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH running AS (\n SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag\n )\n SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\",\n (SELECT count(*) FROM worker_ping w\n WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag]\n ) AS \"workers!\"\n FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag)\n LEFT JOIN running r ON r.tag = t.tag\n ORDER BY t.tag", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "running!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "workers!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "7745127eb4a4be2b67427a708e8e5bf2973af84cf315047e586071438cd5e438" +} diff --git a/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json b/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json new file mode 100644 index 0000000000..c8b20a20ea --- /dev/null +++ b/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base)\n VALUES ($1, $2, $3::text, $4,\n CASE WHEN $10::bool\n THEN to_json($5::text::jsonb || jsonb_strip_nulls(jsonb_build_object(\n 'path', CASE WHEN $5::text::jsonb -> 'path' = to_jsonb($11::text)\n THEN to_jsonb($3::text) END,\n 'draft_path', CASE WHEN $5::text::jsonb -> 'draft_path' = to_jsonb($11::text)\n THEN to_jsonb($3::text) END)))\n ELSE $5::text::json\n END,\n COALESCE($8::timestamptz, now()), $9)\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = CASE\n WHEN NOT $10::bool\n OR position(chr(92) || 'u0000' in replace(draft.value::text, chr(92) || chr(92), '')) > 0\n THEN EXCLUDED.value\n ELSE to_json((to_jsonb(EXCLUDED.value) - 'path' - 'draft_path')\n || jsonb_strip_nulls(jsonb_build_object(\n 'path', to_jsonb(draft.value) -> 'path',\n 'draft_path', to_jsonb(draft.value) -> 'draft_path')))\n END,\n created_at = EXCLUDED.created_at,\n base = EXCLUDED.base\n WHERE $7::bool = true\n OR $6::timestamptz IS NULL\n OR draft.created_at <= $6::timestamptz\n RETURNING path, created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Timestamptz", + "Bool", + "Timestamptz", + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579" +} diff --git a/backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json b/backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json new file mode 100644 index 0000000000..46f8d77060 --- /dev/null +++ b/backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", +<<<<<<<< HEAD:backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, variable_expiration_handler) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, variable_expiration_handler FROM workspace_settings WHERE workspace_id = $2", +======== + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", +>>>>>>>> origin/main:backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, +<<<<<<<< HEAD:backend/.sqlx/query-6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da.json + "hash": "6b40726bdc0d9be7eac7680eeabb0f92341d7ba86acaf43cf8209b45849c43da" +======== + "hash": "77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1" +>>>>>>>> origin/main:backend/.sqlx/query-77d599e4f7c574dffac4824f37127c7ae2ef5f27d665ad9018f5cdea0f6f2cb1.json +} diff --git a/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json b/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json new file mode 100644 index 0000000000..ce3d615eb0 --- /dev/null +++ b/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND old_path IN ($3, $4) AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14" +} diff --git a/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json b/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json new file mode 100644 index 0000000000..c36b6fb361 --- /dev/null +++ b/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id <> $1\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $2\n ORDER BY ws.workspace_id, dt.key\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629" +} diff --git a/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json b/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json new file mode 100644 index 0000000000..98aa0d9367 --- /dev/null +++ b/backend/.sqlx/query-79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text))\n WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "79d3bbd278c34734ce5babc3be852898bb83047f3563f789753e8fec030d301f" +} diff --git a/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json b/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json new file mode 100644 index 0000000000..b7cccbfad4 --- /dev/null +++ b/backend/.sqlx/query-7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check SET github_posted = true\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2\n AND check_run_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7a0ddb6821d8f628bcf85f786e5864e09c7a5a421ba99647570c2b557d53aa51" +} diff --git a/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json new file mode 100644 index 0000000000..ae8cc698b9 --- /dev/null +++ b/backend/.sqlx/query-7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag AS \"tag!\", count AS \"count!\",\n EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\",\n EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\"\n FROM (\n SELECT tag, sum(n)::bigint AS count,\n (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head\n FROM (\n SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head\n FROM v2_job_queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag, priority\n ) g\n GROUP BY tag\n ) t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "delay!", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "head_since!", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null, + null + ] + }, + "hash": "7af0fd3d8dd1d949ce11b190a4fa6b56c84904aeed791b2a0879b36add6bf9b5" +} diff --git a/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json b/backend/.sqlx/query-7b33adb5cf051bc123340982b97a80a96ee43e1c91d9907d50cc2ee14111623e.json similarity index 78% rename from backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json rename to backend/.sqlx/query-7b33adb5cf051bc123340982b97a80a96ee43e1c91d9907d50cc2ee14111623e.json index 7cfe9aa070..541832d45f 100644 --- a/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json +++ b/backend/.sqlx/query-7b33adb5cf051bc123340982b97a80a96ee43e1c91d9907d50cc2ee14111623e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace, workspace.dev_workspace_label,\n workspace.owner AS \"created_by?\",\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace, workspace.dev_workspace_label,\n workspace.owner AS \"created_by?\",\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled, usr.is_service_account\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", "describe": { "columns": [ { @@ -52,6 +52,11 @@ "ordinal": 9, "name": "disabled", "type_info": "Bool" + }, + { + "ordinal": 10, + "name": "is_service_account", + "type_info": "Bool" } ], "parameters": { @@ -69,8 +74,9 @@ true, false, null, + false, false ] }, - "hash": "88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5" + "hash": "7b33adb5cf051bc123340982b97a80a96ee43e1c91d9907d50cc2ee14111623e" } diff --git a/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json new file mode 100644 index 0000000000..c5740cfb88 --- /dev/null +++ b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_environment_state SET script_path = $3\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba" +} diff --git a/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json new file mode 100644 index 0000000000..6a004023a0 --- /dev/null +++ b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version\n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2\n ORDER BY flow_version.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35" +} diff --git a/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json new file mode 100644 index 0000000000..f4bd63c718 --- /dev/null +++ b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630" +} diff --git a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json similarity index 78% rename from backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json rename to backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json index ffd1670c04..9f7be9d7cb 100644 --- a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json +++ b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6" + "hash": "82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b" } diff --git a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json b/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json deleted file mode 100644 index f98f3fdeea..0000000000 --- a/backend/.sqlx/query-82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "82f6674f19e8ad51a992505a46f46fc4a48172f104e9e849f755ac041c3eef92" -} diff --git a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json similarity index 52% rename from backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json rename to backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json index fc354fb9a2..f417ab94f9 100644 --- a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json +++ b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset\n FROM resource_type\n WHERE workspace_id = $1", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name\n FROM resource_type\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986" + "hash": "86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4" } diff --git a/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json b/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json new file mode 100644 index 0000000000..7627d9828d --- /dev/null +++ b/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba" +} diff --git a/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json new file mode 100644 index 0000000000..98e08b8167 --- /dev/null +++ b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e" +} diff --git a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json b/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json deleted file mode 100644 index 8cdb2d5907..0000000000 --- a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email)\n SELECT $2, path, typ,\n CASE WHEN typ IN ('script', 'flow')\n THEN to_json(to_jsonb(value) - 'on_behalf_of')\n ELSE value END,\n created_at, email\n FROM draft\n WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5" -} diff --git a/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json new file mode 100644 index 0000000000..76456ea8fb --- /dev/null +++ b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TimestampArray" + ] + }, + "nullable": [] + }, + "hash": "87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe" +} diff --git a/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json new file mode 100644 index 0000000000..a498ecccd0 --- /dev/null +++ b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912" +} diff --git a/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json b/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json new file mode 100644 index 0000000000..e0d369fd12 --- /dev/null +++ b/backend/.sqlx/query-89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET concluded = true, conclusion = $3, concluded_at = now()\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2 AND NOT concluded\n RETURNING check_run_id, poster_workspace_id, repo_url", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "check_run_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "repo_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + false, + false + ] + }, + "hash": "89a7f413f6f37aeb7e777faeebb0c4e1612928737a5fb2b4ff1336d8827f788b" +} diff --git a/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json b/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json new file mode 100644 index 0000000000..7a7110030c --- /dev/null +++ b/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc" +} diff --git a/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json new file mode 100644 index 0000000000..0bcfd38acd --- /dev/null +++ b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.is_admin, u.operator, u.disabled, EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) AS \"instance_disabled!\" FROM usr u WHERE u.workspace_id = $1 AND u.email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "disabled", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "instance_disabled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013" +} diff --git a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json b/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json deleted file mode 100644 index d0b96444ee..0000000000 --- a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hostname", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "mode", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "worker_group", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "log_ts", - "type_info": "Timestamp" - }, - { - "ordinal": 4, - "name": "file_path", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "ok_lines", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "err_lines", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "json_fmt", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Float8", - "Int8" - ] - }, - "nullable": [ - false, - null, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db" -} diff --git a/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json new file mode 100644 index 0000000000..610030cffc --- /dev/null +++ b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, lock FROM script\n WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "lock", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455" +} diff --git a/backend/.sqlx/query-8ebe054b41793f1a7b85f1f8d29cd21d12fa1207b58bfc46249c250cdcdb5363.json b/backend/.sqlx/query-8ebe054b41793f1a7b85f1f8d29cd21d12fa1207b58bfc46249c250cdcdb5363.json new file mode 100644 index 0000000000..7a822a8d6a --- /dev/null +++ b/backend/.sqlx/query-8ebe054b41793f1a7b85f1f8d29cd21d12fa1207b58bfc46249c250cdcdb5363.json @@ -0,0 +1,232 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n ai_config,\n dbt_warehouses,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts,\n guest_access_enabled,\n guest_jwt_public_key,\n guest_jwt_jwks_url,\n add_admins_and_developers_to_forks\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "teams_team_guid", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "slack_command_script", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "teams_command_script", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "slack_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "customer_id", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "plan", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "webhook", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "ai_config", + "type_info": "Jsonb" + }, + { + "ordinal": 15, + "name": "dbt_warehouses", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 17, + "name": "datatable", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "ducklake", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "git_sync", + "type_info": "Jsonb" + }, + { + "ordinal": 20, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "default_app", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "default_scripts", + "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 24, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 25, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 26, + "name": "git_app_installations", + "type_info": "Jsonb" + }, + { + "ordinal": 27, + "name": "auto_invite", + "type_info": "Jsonb" + }, + { + "ordinal": 28, + "name": "error_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 29, + "name": "success_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "public_app_execution_limit_per_minute", + "type_info": "Int4" + }, + { + "ordinal": 31, + "name": "error_handler_fallback_to_instance_alerts", + "type_info": "Bool" + }, + { + "ordinal": 32, + "name": "guest_access_enabled", + "type_info": "Bool" + }, + { + "ordinal": 33, + "name": "guest_jwt_public_key", + "type_info": "Text" + }, + { + "ordinal": 34, + "name": "guest_jwt_jwks_url", + "type_info": "Text" + }, + { + "ordinal": 35, + "name": "add_admins_and_developers_to_forks", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false, + false, + true, + true, + false + ] + }, + "hash": "8ebe054b41793f1a7b85f1f8d29cd21d12fa1207b58bfc46249c250cdcdb5363" +} diff --git a/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json new file mode 100644 index 0000000000..552a38a0d4 --- /dev/null +++ b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_settings.workspace_id AS \"id!\",\n workspace_settings.ai_config->'sessions_retention_days' AS retention\n FROM workspace_settings\n LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2\n WHERE workspace_settings.workspace_id = ANY($1)\n AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "retention", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff" +} diff --git a/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json b/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json new file mode 100644 index 0000000000..db3362bf0c --- /dev/null +++ b/backend/.sqlx/query-8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ci_test_job_ids\n FROM git_sync_synced_head\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ci_test_job_ids", + "type_info": "UuidArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "8efe5509034327c202cb3fdd409ce00cc1dbd4921a9ccee8f906515286190b8f" +} diff --git a/backend/.sqlx/query-dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103.json b/backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json similarity index 65% rename from backend/.sqlx/query-dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103.json rename to backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json index 4a82795a3c..758da55ad2 100644 --- a/backend/.sqlx/query-dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103.json +++ b/backend/.sqlx/query-9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n j.args as \"args: Json>>\",\n js.flow_status as \"flow_status: Json\"\n FROM v2_job_status js\n INNER JOIN v2_job j ON j.id = js.id\n WHERE js.id = $1\n ", + "query": "\n SELECT\n j.args as \"args: Json>>\",\n js.flow_status as \"flow_status: Json\",\n j.runnable_path\n FROM v2_job_status js\n INNER JOIN v2_job j ON j.id = js.id\n WHERE js.id = $1\n ", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "flow_status: Json", "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "runnable_path", + "type_info": "Varchar" } ], "parameters": { @@ -20,9 +25,10 @@ ] }, "nullable": [ + true, true, true ] }, - "hash": "dd89d652154748d6d7e625e31778f6885d0ee62d29a4b8894a4b459dd215a103" + "hash": "9008f9abb70a9a07e38acb20bea6a710d0efd77dac4aedeb88d72240e816530b" } diff --git a/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json b/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json new file mode 100644 index 0000000000..14a20fa112 --- /dev/null +++ b/backend/.sqlx/query-904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "904c81997a1594ca998a4b1af5a5710bcbe114153f9096d0e2e59e429bd5a325" +} diff --git a/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json b/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json new file mode 100644 index 0000000000..1555f3683d --- /dev/null +++ b/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a" +} diff --git a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json b/backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json similarity index 59% rename from backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json rename to backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json index 3206ba4433..57dbc41a9f 100644 --- a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json +++ b/backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 RETURNING name", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 AND code_up = $4 AND code_down IS NOT DISTINCT FROM $5 RETURNING name", "describe": { "columns": [ { @@ -13,12 +13,14 @@ "Left": [ "Text", "Text", - "Int8" + "Int8", + "Text", + "Text" ] }, "nullable": [ false ] }, - "hash": "3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8" + "hash": "91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d" } diff --git a/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json new file mode 100644 index 0000000000..542d523f53 --- /dev/null +++ b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, email, name, kind, version, created_by, content, shared_at\n FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2\n AND shared_at > now() - ($3::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42" +} diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json deleted file mode 100644 index 823a65cda3..0000000000 --- a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" -} diff --git a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json b/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json new file mode 100644 index 0000000000..5161f716ff --- /dev/null +++ b/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "conversation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352" +} diff --git a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json b/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json deleted file mode 100644 index 733478e0e9..0000000000 --- a/backend/.sqlx/query-96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "96aa1db2859c80d37ffecc2885442ae6d6d4b6731c3f62e7b85ae7a4f1b666a7" -} diff --git a/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json b/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json new file mode 100644 index 0000000000..4704ddd84e --- /dev/null +++ b/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND (old_path = $3 OR (new_path = $3 AND $4::text IS NULL))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5" +} diff --git a/backend/.sqlx/query-255ba68caa78d0e814ea817693a319f769907025f9c7d1150d45e8ecb1bff4ab.json b/backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json similarity index 51% rename from backend/.sqlx/query-255ba68caa78d0e814ea817693a319f769907025f9c7d1150d45e8ecb1bff4ab.json rename to backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json index 245c65a7a8..87666986a3 100644 --- a/backend/.sqlx/query-255ba68caa78d0e814ea817693a319f769907025f9c7d1150d45e8ecb1bff4ab.json +++ b/backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json @@ -1,15 +1,15 @@ { "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET auto_invite = $1 WHERE workspace_id = $2", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()", "describe": { "columns": [], "parameters": { "Left": [ - "Jsonb", - "Text" + "Varchar", + "Jsonb" ] }, "nullable": [] }, - "hash": "255ba68caa78d0e814ea817693a319f769907025f9c7d1150d45e8ecb1bff4ab" + "hash": "975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820" } diff --git a/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json new file mode 100644 index 0000000000..c7db67f4d6 --- /dev/null +++ b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e" +} diff --git a/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json new file mode 100644 index 0000000000..b98a71fb8a --- /dev/null +++ b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26" +} diff --git a/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json b/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json new file mode 100644 index 0000000000..b63a9d294e --- /dev/null +++ b/backend/.sqlx/query-98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_synced_head SET tests_dispatched_at = NULL\n WHERE workspace_id = $1 AND repo_resource_path = $2 AND branch = $3 AND sha = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98b036be15cbd5efbaf2420feb56dd175aeab87f6377f6ac7b51956ce6d5f039" +} diff --git a/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json b/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json new file mode 100644 index 0000000000..a4a692fd4b --- /dev/null +++ b/backend/.sqlx/query-98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb('u/' || $1))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "98cb765a480dac8a27ecad5df26c26f3dd6ff9aa87293ca671be32fd1305a73a" +} diff --git a/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json b/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json new file mode 100644 index 0000000000..9960ba85c9 --- /dev/null +++ b/backend/.sqlx/query-99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "99194b850cb30d174c8d99303f6ed693e011a89aefb3a0e239630e6950dca5cd" +} diff --git a/backend/.sqlx/query-9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0.json b/backend/.sqlx/query-9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0.json new file mode 100644 index 0000000000..30216f04e9 --- /dev/null +++ b/backend/.sqlx/query-9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE av.raw_app = false)::BIGINT AS \"low_code!\",\n COUNT(*) FILTER (WHERE av.raw_app = true)::BIGINT AS \"raw!\"\n FROM app a\n JOIN app_version av ON av.id = a.versions[array_upper(a.versions, 1)]\n WHERE a.policy->>'sandbox' = 'true'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "low_code!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "raw!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "9b727f03e74ea4a35146c9a339cda82104a9ee39bfbe5d932340a3c2c209c5d0" +} diff --git a/backend/.sqlx/query-9bd1995747f0073b3a866d1238e78f7e6dfa9185056f7731547ff05b8176b271.json b/backend/.sqlx/query-9bd1995747f0073b3a866d1238e78f7e6dfa9185056f7731547ff05b8176b271.json new file mode 100644 index 0000000000..9c4d587c01 --- /dev/null +++ b/backend/.sqlx/query-9bd1995747f0073b3a866d1238e78f7e6dfa9185056f7731547ff05b8176b271.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET add_admins_and_developers_to_forks = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9bd1995747f0073b3a866d1238e78f7e6dfa9185056f7731547ff05b8176b271" +} diff --git a/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json new file mode 100644 index 0000000000..6ef7b20a4f --- /dev/null +++ b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e" +} diff --git a/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json b/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json new file mode 100644 index 0000000000..f882ce2986 --- /dev/null +++ b/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND new_path = $4 AND old_path <> $3 AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609" +} diff --git a/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json new file mode 100644 index 0000000000..0a0f8b4732 --- /dev/null +++ b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + null + ] + }, + "hash": "9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032" +} diff --git a/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json new file mode 100644 index 0000000000..49d01c2b8b --- /dev/null +++ b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',\n 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',\n 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82" +} diff --git a/backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json b/backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json new file mode 100644 index 0000000000..4dd7ea9bd8 --- /dev/null +++ b/backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1" +} diff --git a/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json new file mode 100644 index 0000000000..909e6ad42d --- /dev/null +++ b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247" +} diff --git a/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json new file mode 100644 index 0000000000..f4c4fd31de --- /dev/null +++ b/backend/.sqlx/query-a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH update_lock AS (\n UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3\n )\n INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n VALUES ($3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text", + "Varchar", + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "a0ec5048ddb7640b4407013ed45b7a545959c209c550e48bf108734f293e18e4" +} diff --git a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json b/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json deleted file mode 100644 index 3ed4f5a316..0000000000 --- a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "created_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "deployment_msg", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a" -} diff --git a/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json b/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json new file mode 100644 index 0000000000..d8fdd5140b --- /dev/null +++ b/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff" +} diff --git a/backend/.sqlx/query-a1bcf143135ecb9f32709a56a353d3246dc238920c3e155caa9cf767246fa6fb.json b/backend/.sqlx/query-a1bcf143135ecb9f32709a56a353d3246dc238920c3e155caa9cf767246fa6fb.json new file mode 100644 index 0000000000..18ee22083e --- /dev/null +++ b/backend/.sqlx/query-a1bcf143135ecb9f32709a56a353d3246dc238920c3e155caa9cf767246fa6fb.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET auto_invite = (COALESCE(auto_invite, '{}'::jsonb) - 'domain') || $1::jsonb WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a1bcf143135ecb9f32709a56a353d3246dc238920c3e155caa9cf767246fa6fb" +} diff --git a/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json b/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json new file mode 100644 index 0000000000..c536425f1d --- /dev/null +++ b/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json @@ -0,0 +1,85 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n -- The guard's own predicate, ordered own row, then legacy, then another\n -- user's other-kind row: each needs a different remedy, and a bare\n -- LIMIT 1 would name an arbitrary one.\n (SELECT typ::text FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[])\n AND (typ <> $2 OR email = $4 OR email IS NULL)\n ORDER BY CASE WHEN email = $4 THEN 0 WHEN email IS NULL THEN 1 ELSE 2 END\n LIMIT 1) as \"at_target\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[]) AND email = $4) as \"at_target_own!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[]) AND email IS NULL) as \"at_target_legacy!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5\n AND typ = $2 AND email = $4\n AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) > 0\n ) as \"poisoned!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5\n AND typ = $2 AND email IS NULL) as \"legacy!\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "at_target", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "at_target_own!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "at_target_legacy!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "poisoned!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "legacy!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907" +} diff --git a/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json b/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json similarity index 68% rename from backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json rename to backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json index 5889ab4ee7..0f43afed39 100644 --- a/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json +++ b/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -102,8 +107,9 @@ true, false, false, - true + true, + false ] }, - "hash": "b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48" + "hash": "a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a" } diff --git a/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json b/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json new file mode 100644 index 0000000000..e6fd39901e --- /dev/null +++ b/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin, devops, login_type FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "devops", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "login_type", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901" +} diff --git a/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json b/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json new file mode 100644 index 0000000000..5f2bb07e24 --- /dev/null +++ b/backend/.sqlx/query-a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT enabled\n FROM native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3871035319012d72679f132696146d5275cdd4313961c76223ed0f3fec7dca3" +} diff --git a/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json new file mode 100644 index 0000000000..8c6dd010af --- /dev/null +++ b/backend/.sqlx/query-a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH queue_metrics as (\n SELECT id, created_at,\n CASE WHEN jsonb_typeof(value) = 'object'\n THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric)\n ELSE value\n END AS value\n FROM metrics\n WHERE id LIKE 'queue_%'\n AND created_at > now() - interval '14 day'\n )\n SELECT id, array_agg(json_build_object('value', value, 'created_at', created_at) ORDER BY created_at ASC) as \"values!\"\n FROM queue_metrics\n GROUP BY id\n ORDER BY id ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "values!", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "a498f752e169b711c0cb26ac167b2e554a7fdfc1055389fd65f330469e329d16" +} diff --git a/backend/.sqlx/query-e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082.json b/backend/.sqlx/query-a4a823f70b3dbe6aaf4a61c98345e94c5042fd5e6351fea139a66ecb1fb812ab.json similarity index 59% rename from backend/.sqlx/query-e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082.json rename to backend/.sqlx/query-a4a823f70b3dbe6aaf4a61c98345e94c5042fd5e6351fea139a66ecb1fb812ab.json index a3374d6cdf..9ec8c1e2d5 100644 --- a/backend/.sqlx/query-e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082.json +++ b/backend/.sqlx/query-a4a823f70b3dbe6aaf4a61c98345e94c5042fd5e6351fea139a66ecb1fb812ab.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success\n FROM flow_conversation_message\n WHERE conversation_id = $1\n AND created_seq > $2\n ORDER BY created_seq ASC\n LIMIT $3\n ", + "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning, attachments\n FROM (\n SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning, attachments\n FROM flow_conversation_message\n WHERE conversation_id = $1\n ORDER BY created_seq DESC\n LIMIT $2 OFFSET $3\n ) AS messages\n ORDER BY created_seq ASC\n ", "describe": { "columns": [ { @@ -58,6 +58,26 @@ "ordinal": 8, "name": "success", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "tool_arguments", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "tool_result", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "reasoning", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "attachments", + "type_info": "Jsonb" } ], "parameters": { @@ -76,8 +96,12 @@ false, false, true, - false + false, + true, + true, + true, + true ] }, - "hash": "e8802be9203c1e88a06e337260ccca029380139f89a01a89033e36a6ed9ac082" + "hash": "a4a823f70b3dbe6aaf4a61c98345e94c5042fd5e6351fea139a66ecb1fb812ab" } diff --git a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json b/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json deleted file mode 100644 index 4e3b39403f..0000000000 --- a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e" -} diff --git a/backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json b/backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json new file mode 100644 index 0000000000..f3b2cb571e --- /dev/null +++ b/backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f" +} diff --git a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json deleted file mode 100644 index 7a45e6c402..0000000000 --- a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5" -} diff --git a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json deleted file mode 100644 index 4fa871c594..0000000000 --- a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e" -} diff --git a/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json b/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json new file mode 100644 index 0000000000..2acc85fcd4 --- /dev/null +++ b/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07" +} diff --git a/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json b/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json new file mode 100644 index 0000000000..fa43d27ed8 --- /dev/null +++ b/backend/.sqlx/query-a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH survivors AS (\n SELECT value::text AS rendered FROM resource\n WHERE workspace_id = $1 AND NOT (path = ANY($2::text[]))\n )\n SELECT v.path FROM unnest($3::text[]) AS v(path)\n WHERE EXISTS (\n SELECT 1 FROM survivors s\n WHERE strpos(s.rendered, '\"$var:' || v.path || '\"') > 0\n OR strpos(s.rendered, '\"$jsonvar:' || v.path || '\"') > 0\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7b6731427d51eb34744ca6a39e30d02949f149901ec95323b5725911944df4d" +} diff --git a/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json new file mode 100644 index 0000000000..a83c34e0db --- /dev/null +++ b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH retired AS (\n UPDATE log_file SET indexed_at = now()\n WHERE indexed_at IS NULL\n AND log_ts <= NOW() - make_interval(secs => $1) RETURNING 1)\n SELECT count(*) AS \"n!\" FROM retired", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574" +} diff --git a/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json b/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json new file mode 100644 index 0000000000..26301c39b2 --- /dev/null +++ b/backend/.sqlx/query-a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM git_sync_ci_test_check c\n WHERE c.concluded AND c.github_posted\n AND c.concluded_at < now() - make_interval(secs => $1)\n AND NOT EXISTS (\n SELECT 1 FROM git_sync_synced_head h\n WHERE h.workspace_id = c.workspace_id\n AND h.repo_resource_path = c.repo_resource_path\n AND h.sha = c.head_sha\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [] + }, + "hash": "a7d5a7b6b3bb88f5f7926da577f2cc25020b11882fb359b136466c82b040f8a1" +} diff --git a/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json new file mode 100644 index 0000000000..bc12fdba1f --- /dev/null +++ b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator)\n SELECT $1::varchar, u.username, u.email, true, false FROM usr u\n WHERE u.workspace_id = $2 AND u.email = $3 AND u.is_admin AND NOT u.operator AND NOT u.disabled\n AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled)\n AND NOT EXISTS (SELECT 1 FROM usr f WHERE f.workspace_id = $1::varchar AND f.email = $3)\n ON CONFLICT DO NOTHING\n RETURNING username", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8" +} diff --git a/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json new file mode 100644 index 0000000000..313dec3743 --- /dev/null +++ b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock',\n 'model', 'stock', 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily',\n 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c" +} diff --git a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json b/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json similarity index 52% rename from backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json rename to backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json index 24a5ee62dd..258ee9eaa7 100644 --- a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json +++ b/backend/.sqlx/query-a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352.json @@ -1,22 +1,22 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", + "query": "SELECT repo_resource_path, head_sha FROM git_sync_ci_test_check\n WHERE workspace_id = $1 AND NOT concluded", "describe": { "columns": [ { "ordinal": 0, - "name": "file_path", + "name": "repo_resource_path", "type_info": "Varchar" }, { "ordinal": 1, - "name": "hostname", + "name": "head_sha", "type_info": "Varchar" } ], "parameters": { "Left": [ - "Int8" + "Text" ] }, "nullable": [ @@ -24,5 +24,5 @@ false ] }, - "hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033" + "hash": "a970bbf4d3d064614bc47d438a0447eff928c4518fd2f6800145bf1211086352" } diff --git a/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json b/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json new file mode 100644 index 0000000000..ef7411614b --- /dev/null +++ b/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT consumed_at IS NOT NULL AS \"used!\" FROM login_link WHERE token_hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "used!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c" +} diff --git a/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json b/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json new file mode 100644 index 0000000000..5a9e5da58e --- /dev/null +++ b/backend/.sqlx/query-ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598.json @@ -0,0 +1,86 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n workspace_id,\n path,\n route_path,\n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n allowed_origins,\n script_path,\n summary,\n description,\n is_flow,\n mode,\n request_type,\n authentication_method,\n http_method,\n static_asset_config,\n edited_by,\n permissioned_as,\n edited_at,\n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, now(), $21, $22, $23, $24\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Bool", + "TextArray", + "Varchar", + "Varchar", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ab752dd133b20103800554b3f6622e8a9147d36a4b56ebd30a1aaa960156b598" +} diff --git a/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json new file mode 100644 index 0000000000..c3ef22f973 --- /dev/null +++ b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db" +} diff --git a/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json new file mode 100644 index 0000000000..a741ee1c0d --- /dev/null +++ b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, GREATEST(0, $1::bigint), now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_daily_usage.cost_nanos + $1::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [] + }, + "hash": "acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json new file mode 100644 index 0000000000..41481e2ec7 --- /dev/null +++ b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f" +} diff --git a/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json b/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json new file mode 100644 index 0000000000..1bf06d5366 --- /dev/null +++ b/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO cloud_trial_offer (email, created_by) VALUES ($1, $2)\n ON CONFLICT (email) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961" +} diff --git a/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json new file mode 100644 index 0000000000..222bfc0f82 --- /dev/null +++ b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_unlock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_unlock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385" +} diff --git a/backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json b/backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json new file mode 100644 index 0000000000..65c7f47b30 --- /dev/null +++ b/backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws\n WHERE ws.workspace_id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c" +} diff --git a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json deleted file mode 100644 index 8d09036772..0000000000 --- a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "suspend", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - false - ] - }, - "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a" -} diff --git a/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json new file mode 100644 index 0000000000..f67a0f39f2 --- /dev/null +++ b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2) AS \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef" +} diff --git a/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json b/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json new file mode 100644 index 0000000000..2991d30d1c --- /dev/null +++ b/backend/.sqlx/query-b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE git_sync_ci_test_check\n SET github_posted = github_posted\n AND check_run_id IS NOT DISTINCT FROM GREATEST(check_run_id, $3),\n check_run_id = GREATEST(check_run_id, $3)\n WHERE workspace_id = $1 AND repo_resource_path = $4 AND head_sha = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b7c72ecebf6818d4e60a02edb986c920aa2e14365e63ef8c28c7f65da7c6c9ab" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json new file mode 100644 index 0000000000..7476ae9871 --- /dev/null +++ b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json index b336210daf..b892061f56 100644 --- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -98,12 +98,12 @@ null, null, null, - true, + false, null, null, null, - true, - true + false, + false ] }, "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" diff --git a/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json b/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json new file mode 100644 index 0000000000..4b3a2f33e9 --- /dev/null +++ b/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(value->'databases') FROM global_settings\n WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jsonb_object_keys", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6" +} diff --git a/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json deleted file mode 100644 index 94cf77ebc5..0000000000 --- a/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)\n SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via\n FROM usr WHERE workspace_id = $2\n ON CONFLICT DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142" -} diff --git a/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json new file mode 100644 index 0000000000..934742008f --- /dev/null +++ b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',\n 'u/a/wh/analytics/raw_orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24" +} diff --git a/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json b/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json new file mode 100644 index 0000000000..31c628c1a8 --- /dev/null +++ b/backend/.sqlx/query-bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM git_sync_ci_test_check\n WHERE poster_workspace_id = $1 AND repo_resource_path = $2 AND head_sha = $3\n ) as \"exists!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bafee32cbff8bb7fff26a241d9ad203ea689c1e4b094a2c90eb90c8e4b6e0dff" +} diff --git a/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json new file mode 100644 index 0000000000..980ba7a36c --- /dev/null +++ b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(git_sync, '{repositories}',\n COALESCE((SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' IN ($2, $3)\n THEN CASE WHEN $4::jsonb = 'null'::jsonb\n THEN elem - 'credential'\n ELSE jsonb_set(elem, '{credential}', $4) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem), '[]'::jsonb)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n AND EXISTS (\n SELECT 1 FROM jsonb_array_elements(git_sync->'repositories') AS e\n WHERE e->>'git_repo_resource_path' IN ($2, $3)\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c" +} diff --git a/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json b/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json new file mode 100644 index 0000000000..8a59831e8a --- /dev/null +++ b/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a JOIN app_version av\n ON av.id = a.versions[array_upper(a.versions, 1)] AND av.app_id = a.id\n LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee" +} diff --git a/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json b/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json deleted file mode 100644 index f9fbc7a58b..0000000000 --- a/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636" -} diff --git a/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json b/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json new file mode 100644 index 0000000000..d64cf09687 --- /dev/null +++ b/backend/.sqlx/query-bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET policy = jsonb_set(\n jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)),\n ARRAY['on_behalf_of_email'], to_jsonb($4::text)\n ) WHERE policy->>'on_behalf_of' = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bff72874d1fdee7d572e2677aea1dede87b6793f92af884a023253a0ffc3a905" +} diff --git a/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json b/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json deleted file mode 100644 index 774b47f825..0000000000 --- a/backend/.sqlx/query-c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n ))\n RETURNING token_prefix", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "token_prefix", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "c0da3f1f2c55900dbdf92b16ebbfdb7b4cc11a648460f175e4f57d080a0005a5" -} diff --git a/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json b/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json new file mode 100644 index 0000000000..ed6a4f9b5e --- /dev/null +++ b/backend/.sqlx/query-c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO http_trigger (\n path, route_path, route_path_key, script_path, is_flow, workspace_id,\n edited_by, edited_at, extra_perms, authentication_method, http_method,\n static_asset_config, is_static_website, workspaced_route, wrap_body,\n raw_string, allowed_origins, authentication_resource_path, summary, description,\n error_handler_path, error_handler_args, retry, request_type, mode,\n permissioned_as, labels\n )\n SELECT\n path, route_path, route_path_key, script_path, is_flow, $1,\n edited_by, edited_at, extra_perms, authentication_method, http_method,\n static_asset_config, is_static_website, workspaced_route, wrap_body,\n raw_string, allowed_origins, authentication_resource_path, summary, description,\n error_handler_path, error_handler_args, retry, request_type, 'disabled'::TRIGGER_MODE,\n permissioned_as, labels\n FROM http_trigger\n WHERE workspace_id = $2\n AND (workspaced_route IS TRUE OR $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "c1976ac63f5d763b2a747ff92f5ef9db3a0579c889bcae4e8bf33467ce5cebd1" +} diff --git a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json b/backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json similarity index 54% rename from backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json rename to backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json index ca39442f59..f5fde93bf7 100644 --- a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json +++ b/backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json @@ -1,17 +1,17 @@ { "db_name": "PostgreSQL", - "query": "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", + "query": "DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id", "describe": { "columns": [ { "ordinal": 0, - "name": "name", + "name": "username", "type_info": "Varchar" }, { "ordinal": 1, - "name": "description", - "type_info": "Text" + "name": "workspace_id", + "type_info": "Varchar" } ], "parameters": { @@ -24,5 +24,5 @@ false ] }, - "hash": "c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b" + "hash": "c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992" } diff --git a/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json new file mode 100644 index 0000000000..7078c32f1a --- /dev/null +++ b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, $2::bigint, now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = ai_free_token_usage.cost_nanos + $2::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956" +} diff --git a/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json new file mode 100644 index 0000000000..9724a99b18 --- /dev/null +++ b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_sessions_backup_generation = ai_sessions_backup_generation + 1 WHERE workspace_id = $1 AND large_file_storage IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39" +} diff --git a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json deleted file mode 100644 index efd03ae26e..0000000000 --- a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_ids: serde_json::Value", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340" -} diff --git a/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json b/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json new file mode 100644 index 0000000000..c904dc72fd --- /dev/null +++ b/backend/.sqlx/query-c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\"\n FROM git_sync_synced_head h\n JOIN v2_job_completed pc ON pc.id = h.job_id\n JOIN v2_job j ON j.workspace_id = h.workspace_id\n AND j.kind IN ('dependencies', 'flowdependencies', 'appdependencies')\n AND j.created_at >= pc.started_at\n AND j.created_at <= COALESCE(h.tests_dispatched_at, now())\n JOIN v2_job_completed c ON c.id = j.id AND c.status IN ('failure', 'canceled')\n WHERE h.workspace_id = $1 AND h.repo_resource_path = $4\n AND h.branch = $3 AND h.sha = $2\n AND h.source = 'pull'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c38a1cf8d2a8fd89008a98f03ab87a438b619eb1919c028c6c80e972b4ae438d" +} diff --git a/backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json b/backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json new file mode 100644 index 0000000000..5d8ca8dc00 --- /dev/null +++ b/backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n ORDER BY ws.workspace_id, dt.key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba" +} diff --git a/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json b/backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json similarity index 84% rename from backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json rename to backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json index 0af901b2b8..6a9de7e35b 100644 --- a/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json +++ b/backend/.sqlx/query-c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary,\n enabled\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", "describe": { "columns": [ { @@ -68,6 +68,11 @@ "ordinal": 10, "name": "summary", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" } ], "parameters": { @@ -99,8 +104,9 @@ true, false, false, - true + true, + false ] }, - "hash": "15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2" + "hash": "c7766afaea3e187824698cae2b090af9f49ded98cd1824f1ac91cc5dbe709bc1" } diff --git a/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json new file mode 100644 index 0000000000..fee6801a6a --- /dev/null +++ b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n 'u/a/wh/analytics/draft', 'select 3', '{}'),\n ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',\n 'u/a/wh/analytics/draft_src', 'select 4', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f" +} diff --git a/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json b/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json new file mode 100644 index 0000000000..7bc28f0f39 --- /dev/null +++ b/backend/.sqlx/query-c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04.json @@ -0,0 +1,185 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n path,\n script_path,\n is_flow,\n route_path,\n authentication_resource_path,\n workspace_id,\n request_type AS \"request_type: _\",\n authentication_method AS \"authentication_method: _\",\n edited_by,\n permissioned_as,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n allowed_origins,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n mode as \"mode: _\"\n FROM\n http_trigger\n WHERE\n http_method = $1 AND\n (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "authentication_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "request_type: _", + "type_info": { + "Custom": { + "name": "request_type", + "kind": { + "Enum": [ + "sync", + "async", + "sync_sse" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "authentication_method: _", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + }, + { + "ordinal": 8, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "static_asset_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "wrap_body", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "raw_string", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "allowed_origins", + "type_info": "TextArray" + }, + { + "ordinal": 14, + "name": "workspaced_route", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "is_static_website", + "type_info": "Bool" + }, + { + "ordinal": 16, + "name": "error_handler_path", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "error_handler_args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "retry: _", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "mode: _", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + true, + false, + false, + true, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "c7a78d3db99e7f709479c9520471eaf40862b464af7113d2632c626e43c35c04" +} diff --git a/backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json b/backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json new file mode 100644 index 0000000000..7a7da4308d --- /dev/null +++ b/backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_role WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587" +} diff --git a/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json new file mode 100644 index 0000000000..19dc4781a6 --- /dev/null +++ b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n SELECT $1, * FROM UNNEST($2::text[], $3::bigint[])\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash\n WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a" +} diff --git a/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json new file mode 100644 index 0000000000..58300795af --- /dev/null +++ b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem\n FROM workspace_settings, jsonb_array_elements(git_credentials) AS elem\n WHERE workspace_id = $1 AND elem->>'repo_identity' = $2\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15" +} diff --git a/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json new file mode 100644 index 0000000000..2ef26b366b --- /dev/null +++ b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 100000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba" +} diff --git a/backend/.sqlx/query-cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2.json b/backend/.sqlx/query-cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2.json new file mode 100644 index 0000000000..9ff8b3ace5 --- /dev/null +++ b/backend/.sqlx/query-cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM guest_activity WHERE day < CURRENT_DATE - 60", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "cf0c44d83ec921d104bee9cbdb7acd7ec38166533d217b718b294826889145f2" +} diff --git a/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json b/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json new file mode 100644 index 0000000000..cb87d3b3f6 --- /dev/null +++ b/backend/.sqlx/query-cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args->>'repo_url_resource_path' FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cfbe6784b3d108f935ba884fe2d3b36afde836fbc6db046279b015bea6f70201" +} diff --git a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json similarity index 72% rename from backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json rename to backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json index b31e532fc2..2947dd8413 100644 --- a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json +++ b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4" + "hash": "cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f" } diff --git a/backend/.sqlx/query-d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a.json b/backend/.sqlx/query-d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a.json new file mode 100644 index 0000000000..2ff2722f77 --- /dev/null +++ b/backend/.sqlx/query-d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $2\n THEN jsonb_set(r.v, '{governed_by,workspace_id}', to_jsonb($1::text))\n ELSE r.v END\n FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $2\n THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))\n ELSE dt.value END AS v) r)\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND (ws.datatable::text LIKE '%\"reference\"%'\n OR ws.datatable::text LIKE '%\"governed_by\"%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a" +} diff --git a/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json new file mode 100644 index 0000000000..cbcb7f18f7 --- /dev/null +++ b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_lock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749" +} diff --git a/backend/.sqlx/query-a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726.json b/backend/.sqlx/query-d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a.json similarity index 65% rename from backend/.sqlx/query-a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726.json rename to backend/.sqlx/query-d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a.json index d507609d8e..c3134373f6 100644 --- a/backend/.sqlx/query-a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726.json +++ b/backend/.sqlx/query-d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE token SET scopes = $1\n WHERE email = $2 AND token_prefix = $3\n RETURNING token_prefix", + "query": "UPDATE token SET scopes = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR label <> 'guest_session')\n RETURNING token_prefix", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "a7a20412e303568b271f949642de55e9880ef05786fe59f05de5e025ef315726" + "hash": "d17645b5001d7f8da1dc451c5d35ea3c9346271b8404863256071cfdf884036a" } diff --git a/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json new file mode 100644 index 0000000000..1e56d65715 --- /dev/null +++ b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact\n WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3\n AND shared_at > now() - ($4::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57" +} diff --git a/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json new file mode 100644 index 0000000000..9c75f7ac97 --- /dev/null +++ b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c" +} diff --git a/backend/.sqlx/query-d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590.json b/backend/.sqlx/query-d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590.json new file mode 100644 index 0000000000..ad5a3c9f56 --- /dev/null +++ b/backend/.sqlx/query-d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT usage_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM asset WHERE kind = 'datatable' AND usage_kind <> 'job'\n GROUP BY 1\n UNION ALL\n SELECT 'job_recent'::text, COUNT(DISTINCT (workspace_id, path))::BIGINT\n FROM asset\n WHERE kind = 'datatable' AND usage_kind = 'job'\n AND created_at > now() - interval '30 days'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "d3a6a27ece3b5d5071dd8074b8db6f369eceb079b31ca118cae23c3d15314590" +} diff --git a/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json new file mode 100644 index 0000000000..49875acd78 --- /dev/null +++ b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id = $1\n AND dt.key <> $2\n AND NOT dt.value ? 'permissions'\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $3\n ORDER BY dt.key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40" +} diff --git a/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json new file mode 100644 index 0000000000..3f73c2171e --- /dev/null +++ b/backend/.sqlx/query-d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO metrics (id, value)\n SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value)\n FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "JsonbArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "d4ce900b8e60b530c2ea57c082edbc8c12c71869c4e45219dad4fb7198aaa1e3" +} diff --git a/backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json b/backend/.sqlx/query-d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e.json similarity index 54% rename from backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json rename to backend/.sqlx/query-d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e.json index d05c5b11af..edc0efd9ca 100644 --- a/backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json +++ b/backend/.sqlx/query-d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", operator as operator_only, is_admin as is_workspace_admin, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2", + "query": "WITH active_users AS (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", operator as operator_only, is_admin as is_workspace_admin, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -97,5 +97,5 @@ null ] }, - "hash": "8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000" + "hash": "d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e" } diff --git a/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json b/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json new file mode 100644 index 0000000000..30cc0e7092 --- /dev/null +++ b/backend/.sqlx/query-d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)))\n WHERE typ IN ('app', 'raw_app')\n AND value->'policy'->>'on_behalf_of' = $2\n AND value->'policy'->>'on_behalf_of_email' = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d599e8058e96f3708cf6af2cc1a2ea3d912ec703687f6ce52d2419a410b1599e" +} diff --git a/backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json b/backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json new file mode 100644 index 0000000000..6297c4d366 --- /dev/null +++ b/backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf" +} diff --git a/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json b/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json new file mode 100644 index 0000000000..70fd38193c --- /dev/null +++ b/backend/.sqlx/query-d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path FROM script WHERE workspace_id = $1 AND deleted = false AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d644b9cd3407e58f235cc2e97558257c07785c8b3123f13d59ee361b4ee0bc0a" +} diff --git a/backend/.sqlx/query-1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051.json b/backend/.sqlx/query-d6fa78c43b6c5f8040d7bccb29ad8627be1dac6fbe0097735a52f47c173f51c9.json similarity index 65% rename from backend/.sqlx/query-1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051.json rename to backend/.sqlx/query-d6fa78c43b6c5f8040d7bccb29ad8627be1dac6fbe0097735a52f47c173f51c9.json index fb27bd9446..baf0e51d4f 100644 --- a/backend/.sqlx/query-1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051.json +++ b/backend/.sqlx/query-d6fa78c43b6c5f8040d7bccb29ad8627be1dac6fbe0097735a52f47c173f51c9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success\n FROM (\n SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success\n FROM flow_conversation_message\n WHERE conversation_id = $1\n ORDER BY created_seq DESC\n LIMIT $2 OFFSET $3\n ) AS messages\n ORDER BY created_seq ASC\n ", + "query": "SELECT id, conversation_id, message_type as \"message_type: MessageType\", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning, attachments\n FROM flow_conversation_message\n WHERE conversation_id = $1\n AND created_seq > $2\n ORDER BY created_seq ASC\n LIMIT $3\n ", "describe": { "columns": [ { @@ -58,6 +58,26 @@ "ordinal": 8, "name": "success", "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "tool_arguments", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "tool_result", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "reasoning", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "attachments", + "type_info": "Jsonb" } ], "parameters": { @@ -76,8 +96,12 @@ false, false, true, - false + false, + true, + true, + true, + true ] }, - "hash": "1c3473a0f9f6b6148b2c975f9f05bdefedf8a51c4e6ddf0eca367b9cc778d051" + "hash": "d6fa78c43b6c5f8040d7bccb29ad8627be1dac6fbe0097735a52f47c173f51c9" } diff --git a/backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json b/backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json deleted file mode 100644 index 70872a1328..0000000000 --- a/backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1" -} diff --git a/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json b/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json new file mode 100644 index 0000000000..b0c2d95175 --- /dev/null +++ b/backend/.sqlx/query-d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d88b1c445acc5375e9c543dcec81059d0e7018a6cea79ba743693098d223edf5" +} diff --git a/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json new file mode 100644 index 0000000000..748ca6d9ec --- /dev/null +++ b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac" +} diff --git a/backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json b/backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json new file mode 100644 index 0000000000..4e63a2afa6 --- /dev/null +++ b/backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1" +} diff --git a/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json new file mode 100644 index 0000000000..d64f007d7b --- /dev/null +++ b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mode!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "log_ts", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d" +} diff --git a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json b/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json similarity index 52% rename from backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json rename to backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json index 121d7fe04a..669e78201d 100644 --- a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json +++ b/backend/.sqlx/query-dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc.json @@ -1,22 +1,22 @@ { "db_name": "PostgreSQL", - "query": "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", + "query": "SELECT is_admin, operator, email FROM usr where username = $1 AND workspace_id = $2 AND disabled = false", "describe": { "columns": [ { "ordinal": 0, - "name": "name", - "type_info": "Varchar" + "name": "is_admin", + "type_info": "Bool" }, { "ordinal": 1, - "name": "description", - "type_info": "Text" + "name": "operator", + "type_info": "Bool" }, { "ordinal": 2, - "name": "instructions", - "type_info": "Text" + "name": "email", + "type_info": "Varchar" } ], "parameters": { @@ -31,5 +31,5 @@ false ] }, - "hash": "e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d" + "hash": "dada076fe622e9902606bf95f9e6df004f8cb7091588309a1f484b91fe7183fc" } diff --git a/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json b/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json new file mode 100644 index 0000000000..02d889c38b --- /dev/null +++ b/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors", + "type_info": "VarcharArray" + }, + { + "ordinal": 1, + "name": "operators", + "type_info": "VarcharArray" + }, + { + "ordinal": 2, + "name": "author_count", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "operator_count", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c" +} diff --git a/backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json b/backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json new file mode 100644 index 0000000000..def9ee0edb --- /dev/null +++ b/backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->'roles' FROM global_settings WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04" +} diff --git a/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json new file mode 100644 index 0000000000..eba1b0da99 --- /dev/null +++ b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a" +} diff --git a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json similarity index 64% rename from backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json rename to backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json index 400c8d9ee6..340b09ca27 100644 --- a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json +++ b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Text", "Varchar", "Varchar", - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00" + "hash": "dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606" } diff --git a/backend/.sqlx/query-dd84f9dfb238d18cb74f9e43228345427131bf8008eba6920021d6a449791534.json b/backend/.sqlx/query-dd84f9dfb238d18cb74f9e43228345427131bf8008eba6920021d6a449791534.json new file mode 100644 index 0000000000..a73c736a1d --- /dev/null +++ b/backend/.sqlx/query-dd84f9dfb238d18cb74f9e43228345427131bf8008eba6920021d6a449791534.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2\n FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "flow_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "title", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_test", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "dd84f9dfb238d18cb74f9e43228345427131bf8008eba6920021d6a449791534" +} diff --git a/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json new file mode 100644 index 0000000000..9622e63a56 --- /dev/null +++ b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id, ws.git_sync\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE NOT w.deleted\n AND ws.git_sync IS NOT NULL\n AND jsonb_typeof(ws.git_sync->'repositories') = 'array'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b" +} \ No newline at end of file diff --git a/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json new file mode 100644 index 0000000000..6ffd197e90 --- /dev/null +++ b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_fileset, format_extension FROM resource_type\n WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fileset", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "format_extension", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a" +} diff --git a/backend/.sqlx/query-dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290.json b/backend/.sqlx/query-dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290.json new file mode 100644 index 0000000000..7acdcf8801 --- /dev/null +++ b/backend/.sqlx/query-dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH capped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n ORDER BY timestamp DESC\n LIMIT 200\n ), uncapped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n ORDER BY timestamp DESC\n LIMIT 200\n )\n SELECT timestamp AS \"timestamp!\",\n operation::text AS \"operation!\",\n resource AS workspace_id,\n parameters\n FROM (SELECT * FROM capped UNION ALL SELECT * FROM uncapped) e\n ORDER BY timestamp DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp!", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "operation!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "parameters", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290" +} diff --git a/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json new file mode 100644 index 0000000000..0b17793580 --- /dev/null +++ b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage IS NOT NULL AS \"has_storage!\" FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_storage!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a" +} diff --git a/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json b/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json new file mode 100644 index 0000000000..9129b657d3 --- /dev/null +++ b/backend/.sqlx/query-dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as \"count!\" FROM v2_job WHERE id = ANY($1::uuid[])", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfffd6573a1eab11c0515805f85b95a942f71651a4f34e671f25ba6816a506c8" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json new file mode 100644 index 0000000000..ae893b6791 --- /dev/null +++ b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" +} diff --git a/backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json b/backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json new file mode 100644 index 0000000000..3f7245f35f --- /dev/null +++ b/backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250" +} diff --git a/backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json b/backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json new file mode 100644 index 0000000000..0e3c0182a9 --- /dev/null +++ b/backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json new file mode 100644 index 0000000000..fc42c451f8 --- /dev/null +++ b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a" +} diff --git a/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json b/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json new file mode 100644 index 0000000000..57b5e2abd1 --- /dev/null +++ b/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET email = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd" +} diff --git a/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json new file mode 100644 index 0000000000..3068e4fe35 --- /dev/null +++ b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool)\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51" +} diff --git a/backend/.sqlx/query-e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa.json b/backend/.sqlx/query-e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa.json new file mode 100644 index 0000000000..af8d0eb1ab --- /dev/null +++ b/backend/.sqlx/query-e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE EXISTS (\n SELECT 1 FROM (VALUES ('reference'), ('governed_by')) k(link)\n WHERE dt.value->k.link->>'workspace_id' = $1\n AND dt.value->k.link->>'datatable' = $2\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa" +} diff --git a/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json new file mode 100644 index 0000000000..797fee956b --- /dev/null +++ b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77" +} diff --git a/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json b/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json new file mode 100644 index 0000000000..f218653d46 --- /dev/null +++ b/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email, base)\n SELECT $2, path, typ,\n to_json(\n CASE WHEN typ IN ('script', 'flow') THEN clean - 'on_behalf_of' ELSE clean END\n - CASE WHEN typ = 'flow' THEN 'version_id'\n WHEN typ IN ('app', 'raw_app') THEN 'parent_version'\n ELSE '' END\n ),\n created_at, email,\n CASE WHEN typ = 'script' THEN base END\n FROM (\n SELECT d.path, d.typ, d.created_at, d.email, d.base,\n replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean\n FROM draft d\n WHERE d.workspace_id = $1 AND (d.email = $3 OR d.email IS NULL)\n ) s", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b" +} diff --git a/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json b/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json new file mode 100644 index 0000000000..926add8159 --- /dev/null +++ b/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT m.new_path FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ = $2 AND m.old_path = $3\n AND (m.email IS NULL OR m.email = $4)\n AND NOT EXISTS (\n SELECT 1 FROM draft d\n WHERE d.workspace_id = $1 AND d.typ = $2 AND d.path = $3\n AND d.email IS NOT DISTINCT FROM $4\n )\n ORDER BY m.email IS NULL\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "new_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2" +} diff --git a/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json b/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json new file mode 100644 index 0000000000..cf4d577dbb --- /dev/null +++ b/backend/.sqlx/query-e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, poster_workspace_id, head_sha, repo_url, repo_resource_path,\n check_run_id\n FROM git_sync_ci_test_check\n WHERE NOT concluded OR NOT github_posted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "head_sha", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "repo_url", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "repo_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "check_run_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + true + ] + }, + "hash": "e64ec4941cbbee016c14d958b7220ddfc414c7e741a171fb80673c23644e3619" +} diff --git a/backend/.sqlx/query-e6e31fdf705896c81f9a0f45d47c1b93db0406aaf278bccba07be00e9f937e2a.json b/backend/.sqlx/query-e6e31fdf705896c81f9a0f45d47c1b93db0406aaf278bccba07be00e9f937e2a.json new file mode 100644 index 0000000000..7462224850 --- /dev/null +++ b/backend/.sqlx/query-e6e31fdf705896c81f9a0f45d47c1b93db0406aaf278bccba07be00e9f937e2a.json @@ -0,0 +1,88 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n slack_name,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n mute_critical_alerts,\n guest_access_enabled,\n add_admins_and_developers_to_forks,\n deploy_ui,\n large_file_storage,\n datatable\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "teams_team_guid", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "guest_access_enabled", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "add_admins_and_developers_to_forks", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "datatable", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + true + ] + }, + "hash": "e6e31fdf705896c81f9a0f45d47c1b93db0406aaf278bccba07be00e9f937e2a" +} diff --git a/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json b/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json new file mode 100644 index 0000000000..f8c42256bb --- /dev/null +++ b/backend/.sqlx/query-e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM git_sync_synced_head h\n WHERE h.synced_at < now() - make_interval(secs => $1)\n AND EXISTS (\n SELECT 1 FROM git_sync_synced_head n\n WHERE n.workspace_id = h.workspace_id\n AND n.repo_resource_path = h.repo_resource_path\n AND n.branch = h.branch\n AND n.synced_at > h.synced_at\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [] + }, + "hash": "e6f2a6fa47bf3b5c774d6bc6060ca5a99addffab3dd44529ce9f40de115b0a3c" +} diff --git a/backend/.sqlx/query-e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef.json b/backend/.sqlx/query-e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef.json new file mode 100644 index 0000000000..328465fdf1 --- /dev/null +++ b/backend/.sqlx/query-e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by, canceled_reason,\n (extract(epoch FROM now() - started_at) * 1000)::bigint AS segment_ms\n FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "segment_ms", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + null + ] + }, + "hash": "e749663a4b9248a9d120f77e86fd3413ad3c0fbe8f3dca9aea11cb49fd4a28ef" +} diff --git a/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json b/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json new file mode 100644 index 0000000000..5d0e560ced --- /dev/null +++ b/backend/.sqlx/query-ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(jc.status::text, 'running') as \"status!\"\n FROM unnest($1::uuid[]) AS run(id)\n LEFT JOIN v2_job_completed jc ON jc.id = run.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ea397add5eb6555457883e5b6bdc67efbe6b0559adb891dba65d8b8e1430f357" +} diff --git a/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json new file mode 100644 index 0000000000..a344e64392 --- /dev/null +++ b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_sessions_backup_generation", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11" +} diff --git a/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json new file mode 100644 index 0000000000..854ed61857 --- /dev/null +++ b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT n.script_path AS \"script_path!\", n.script_hash, n.job_id AS \"job_id!\"\n FROM dbt_node n\n WHERE n.workspace_id = $1 AND n.asset_path = ANY($2)\n -- The run's snapshot, or the deployed graph when that job stored\n -- none -- a build pins only if it wrote one.\n AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $5)\n THEN $5::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END\n -- The gate, re-decided for every project the walk reaches. That\n -- is what resolving owners in a loop is for: being entitled to\n -- one project is not being entitled to the one that declares a\n -- relation it hands over.\n AND ( $6\n OR n.script_path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE n.script_path = pfx\n OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) )\n AND CASE\n -- Pinned: which version comes from a job this caller was\n -- already granted, so `script` does not decide THAT -- but\n -- it still decides whether the project may be read, the\n -- same second gate `script_visible` is on the graph. Being\n -- entitled to a run is not being entitled to the SQL\n -- behind it, and column lineage is that SQL's shape. A\n -- version-less row is exempt because it is an editor\n -- buffer, which has no `script` row to ask and reaches\n -- this only through the parse job that wrote it.\n --\n -- One project answers, so a pinned trace never crosses\n -- into another: neither does the graph it annotates.\n WHEN $4::text IS NOT NULL\n THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint\n AND ($3::bigint IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3))\n -- A named version: the deployed one an editor is drawing.\n -- `script` is read under RLS, so this is the visibility\n -- check as well as the existence one. A hash names one\n -- script row, so this arm answers for one project too —\n -- and deliberately: a pin says which stored graph is on\n -- screen, and another project's live graph is not it.\n WHEN $3::bigint IS NOT NULL\n THEN n.script_hash = $3 AND EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3)\n -- Otherwise the version deployed now: an older one's rows\n -- outlive it in `dbt_node` until the sweep, and describe a\n -- project that is no longer what runs. `language` narrows\n -- it the way the graph's own resolution does, so a path\n -- that has since become a script of another kind draws and\n -- explains the same version rather than disagreeing. Read\n -- under RLS, so a project the caller cannot see resolves\n -- to NULL and matches nothing.\n ELSE n.script_hash = (\n SELECT sc.hash FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.language = 'dbt'\n AND sc.deleted = false AND sc.archived = false\n ORDER BY sc.created_at DESC LIMIT 1)\n END", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8", + "Text", + "Uuid", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2" +} diff --git a/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json b/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json new file mode 100644 index 0000000000..e5de67e73f --- /dev/null +++ b/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(\n datatable #- ARRAY['datatables', $2, 'reference'],\n ARRAY['datatables', $2, 'database'], $3::jsonb),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241" +} diff --git a/backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json b/backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json new file mode 100644 index 0000000000..1e35bd15ac --- /dev/null +++ b/backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM flow_conversation WHERE id = ANY($1) ORDER BY id FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410" +} diff --git a/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json new file mode 100644 index 0000000000..bea6be2a7f --- /dev/null +++ b/backend/.sqlx/query-ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, lock_error_logs, created_at)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, $6::text, clock_timestamp()\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ede523c0b0027f7bc1dacd3a5448031783fa7fbbbf86a291b6f8f8f875d45637" +} diff --git a/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json b/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json new file mode 100644 index 0000000000..ec641bf32d --- /dev/null +++ b/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30" +} diff --git a/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json b/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json new file mode 100644 index 0000000000..3b89a91024 --- /dev/null +++ b/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780" +} diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json new file mode 100644 index 0000000000..5f86fb97d8 --- /dev/null +++ b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" +} diff --git a/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json new file mode 100644 index 0000000000..32f7e8cf1f --- /dev/null +++ b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, GREATEST(0, $2::bigint), now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_usage.cost_nanos + $2::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f" +} diff --git a/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json b/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json new file mode 100644 index 0000000000..7020c21b0d --- /dev/null +++ b/backend/.sqlx/query-f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO git_sync_synced_head\n (workspace_id, repo_resource_path, branch, sha, source, job_id)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, repo_resource_path, branch, sha)\n DO UPDATE SET source = EXCLUDED.source, job_id = EXCLUDED.job_id, synced_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f1282393a95b499f1a9fce5939205879d507b9697eed7e5e232f8e6cb95c2bd0" +} diff --git a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json similarity index 51% rename from backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json rename to backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json index 7df22ca7b7..976cdfec06 100644 --- a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json +++ b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b" + "hash": "f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f" } diff --git a/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json new file mode 100644 index 0000000000..1a71c74e9e --- /dev/null +++ b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25" +} diff --git a/backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json b/backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json new file mode 100644 index 0000000000..64183afaad --- /dev/null +++ b/backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id FROM workspace_settings WHERE datatable::text LIKE $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef" +} diff --git a/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json b/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json new file mode 100644 index 0000000000..cce644b300 --- /dev/null +++ b/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT plan FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "plan", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583" +} diff --git a/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json new file mode 100644 index 0000000000..24f957a4e2 --- /dev/null +++ b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM draft ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691" +} diff --git a/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json new file mode 100644 index 0000000000..75526d2db8 --- /dev/null +++ b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b" +} diff --git a/backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json b/backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json new file mode 100644 index 0000000000..b3dd09e2ba --- /dev/null +++ b/backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = CASE WHEN $3::jsonb = 'null'::jsonb\n THEN datatable #- ARRAY['datatables', $2, 'permissions']\n ELSE jsonb_set(datatable, ARRAY['datatables', $2, 'permissions'], $3::jsonb)\n END\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e" +} diff --git a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json deleted file mode 100644 index 72acab6120..0000000000 --- a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20" -} diff --git a/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json new file mode 100644 index 0000000000..1c79a53ab0 --- /dev/null +++ b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14" +} diff --git a/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json b/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json new file mode 100644 index 0000000000..dda976de02 --- /dev/null +++ b/backend/.sqlx/query-f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT repo_url, check_run_id, poster_workspace_id, conclusion,\n created_at, concluded, github_posted, head_ref\n FROM git_sync_ci_test_check\n WHERE workspace_id = $1 AND repo_resource_path = $3 AND head_sha = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repo_url", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "check_run_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "poster_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "conclusion", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "concluded", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "github_posted", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "head_ref", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "f6a2a8fbc22c69fd5da86626372f84d0ec7b6cb9375e30b10415605cba9b2fcb" +} diff --git a/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json new file mode 100644 index 0000000000..ccb9863853 --- /dev/null +++ b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',\n 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e" +} diff --git a/backend/.sqlx/query-f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6.json b/backend/.sqlx/query-f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6.json new file mode 100644 index 0000000000..d9b9aabf58 --- /dev/null +++ b/backend/.sqlx/query-f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "guest_jwt_public_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "guest_jwt_jwks_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f6fe63ef3518d2d1f321c2941bc59da15843f466c72159bf76712b124d7554b6" +} diff --git a/backend/.sqlx/query-f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6.json b/backend/.sqlx/query-f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6.json new file mode 100644 index 0000000000..1be51feb85 --- /dev/null +++ b/backend/.sqlx/query-f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $1\n AND r.v->'governed_by'->>'datatable' = $2\n THEN jsonb_set(r.v, '{governed_by,datatable}', to_jsonb($3::text))\n ELSE r.v END\n FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END AS v) r)\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE EXISTS (\n SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d,\n LATERAL (VALUES ('reference'), ('governed_by')) k(link)\n WHERE d.value->k.link->>'workspace_id' = $1\n AND d.value->k.link->>'datatable' = $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6" +} diff --git a/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json new file mode 100644 index 0000000000..540d2e0e08 --- /dev/null +++ b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2" +} diff --git a/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json b/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json new file mode 100644 index 0000000000..bf8fa238a0 --- /dev/null +++ b/backend/.sqlx/query-fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE native_trigger\n SET enabled = $1\n WHERE\n workspace_id = $2\n AND service_name = $3\n AND external_id = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text", + { + "Custom": { + "name": "native_trigger_service", + "kind": { + "Enum": [ + "nextcloud", + "google", + "github" + ] + } + } + }, + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa42f89d9494056e1e7ad904b844203f1132aa5ae5eff9194c45e0259f4bee76" +} diff --git a/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json new file mode 100644 index 0000000000..6a4b3fe6de --- /dev/null +++ b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71" +} diff --git a/backend/.sqlx/query-fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1.json b/backend/.sqlx/query-fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1.json new file mode 100644 index 0000000000..20b5c53392 --- /dev/null +++ b/backend/.sqlx/query-fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET guest_jwt_public_key = $1, guest_jwt_jwks_url = $2 WHERE workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fbee9545c564f6fd611ca1a122cf420b03fd23304f78c382d6de14cb31bcd6b1" +} diff --git a/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json b/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json new file mode 100644 index 0000000000..27cb37b876 --- /dev/null +++ b/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT d.email IS NULL as \"legacy!\", COALESCE(u.username, p.username) as username\n FROM draft d\n LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email\n LEFT JOIN password p ON p.email = d.email AND p.super_admin = true\n WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ::text = ANY($3::text[])\n -- A row a move already brought here from the very path being renamed is this\n -- rename's own draft, not an item in its way: the record says which owners\n -- it moved (all of them for an item move, one for a draft-only move). Any\n -- other row is a second item and still collides.\n AND NOT EXISTS (\n SELECT 1 FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ::text = ANY($3::text[])\n AND m.old_path = $4 AND m.new_path = $2\n AND (m.email IS NULL OR m.email IS NOT DISTINCT FROM d.email)\n )\n ORDER BY 2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "legacy!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46" +} diff --git a/backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json b/backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json new file mode 100644 index 0000000000..26920a2943 --- /dev/null +++ b/backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE datatable_role SET name = $2, enabled = $3, pwd = $4 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363" +} diff --git a/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json new file mode 100644 index 0000000000..c1eb5fa826 --- /dev/null +++ b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 7bd84dfca3..750e0594e1 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -23,7 +23,7 @@ ## Cargo features & running the dev backend The dev backend runs under `cargo watch` and is launched by default with **only -`--features quickjs`** (see the tmux backend pane). That baseline compiles fast but +`--features quickjs`** (see the backend pane). That baseline compiles fast but **deliberately omits most functionality** — notably S3/object storage, the S3 proxy, all EE code, MCP, and every non-JS language runtime. A running server never gains a feature you didn't compile in: feature-gated routes 404 or return a `"requires "` stub. So if @@ -33,22 +33,30 @@ MUST **restart the backend with the appropriate features** for what you're worki ### Restarting the dev backend with the right features Restart in the **same pane**, so the relaunch inherits that pane's `DATABASE_URL`, `BACKEND_PORT` -and the rest of `runtime.env`. Scope every kill to this worktree — **never** +and the rest of the worktree environment. Scope every kill to this worktree — **never** `pkill -f target/debug/windmill`, which kills every sibling worktree's backend. -1. Find the backend pane by what it is running, not by index. The index depends on the webmux - profile: pane 1 is the backend under `full`, but the *frontend* under `frontendOnly`. +1. Find the backend pane by where it is running, not by index. Under herdr the two server panes + sit in the worktree's `backend/` and `frontend/`, so match on `cwd`: + + ```bash + herdr pane list --workspace "$HERDR_WORKSPACE_ID" + ``` + + Under webmux they are tmux panes, and the index depends on the profile: pane 1 is the backend + under `full`, but the *frontend* under `frontendOnly`. ```bash WIN=$(tmux display-message -p -t "$TMUX_PANE" '#{window_id}') tmux list-panes -t "$WIN" -F '#{pane_index} #{pane_current_command} #{pane_pid}' ``` -2. Read the feature set it is **actually** running. `CARGO_FEATURES` in `runtime.env` is only what - the pane started with, and goes stale the first time anyone restarts by hand: +2. Read the feature set it is **actually** running. The `CARGO_FEATURES` the worktree was created + with is only what the pane started with, and goes stale the first time anyone restarts by hand: ```bash - ps --ppid -o args= + herdr pane process-info --pane # foreground_processes[].argv + ps --ppid -o args= # under webmux # /home/hugo/.cargo/bin/cargo-watch watch -x run --features quickjs ``` @@ -56,22 +64,25 @@ and the rest of `runtime.env`. Scope every kill to this worktree — **never** explicitly: ```bash - tmux send-keys -t "$WIN." C-c - tmux send-keys -t "$WIN." 'PORT=$BACKEND_PORT cargo watch -x "run --features quickjs,private,parquet"' Enter + herdr pane send-keys C-c + herdr pane run 'PORT=$BACKEND_PORT cargo watch -x "run --features quickjs,private,parquet"' + # under webmux: tmux send-keys -t "$WIN." C-c, then the same command line with Enter ``` Carry over every feature the old command had unless you mean to drop one — rebuilding the list from memory is how a backend silently loses `quickjs`. -4. Persist the new set so a recreated pane starts with it: set `CARGO_FEATURES` in the - worktree's `.env.local`, which `scripts/post-create.sh` writes and webmux reads. Do **not** - edit `runtime.env` for this — webmux regenerates it from metadata and `.env.local` every time - the worktree is opened, so an edit there is lost on the next reopen. Either way the change - only affects a future pane; step 3 is what takes effect now. +4. Persist the new set so a recreated pane starts with it: set `CARGO_FEATURES` in the worktree's + `.env.local`, which the worktree hooks write. A herdr pane sources that file directly, so the + next pane picks the value up. A webmux pane sources `webmux/runtime.env` instead, which webmux + regenerates from metadata and `.env.local` when the worktree is *opened* — so the value lands + on the next reopen, not the next pane, and editing `runtime.env` by hand is lost at that same + reopen. Either way the change only affects a future pane; step 3 is what takes effect now. -5. Re-capture the pane until `health check completed` appears before hitting the API. A cold - rebuild takes ~60s, and the previous run's success line is still in the scrollback, so a - capture taken too early reads as ready when it isn't. +5. Re-read the pane (`herdr pane read --source recent-unwrapped`, or `tmux + capture-pane`) until `health check completed` appears before hitting the API. A cold rebuild + takes ~60s, and the previous run's success line is still in the scrollback, so a capture taken + too early reads as ready when it isn't. cargo-watch only re-runs on a file change, so after an idle/failed run `touch README.md` (from `backend/`, where the watch runs) is a cheap retrigger (touching a `.rs` forces a full rebuild). diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0701814b4b..9a25928de4 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -261,13 +261,13 @@ dependencies = [ [[package]] name = "argon2" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" dependencies = [ "base64ct", - "blake2", - "cpufeatures 0.2.17", + "blake2 0.11.0", + "cpufeatures 0.3.1", "password-hash", ] @@ -430,7 +430,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.14.0", + "indexmap 2.14.2", "lexical-core", "memchr", "num", @@ -549,7 +549,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -561,7 +561,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -728,7 +728,7 @@ dependencies = [ "futures-lite 2.6.1", "parking", "polling 3.11.0", - "rustix 1.1.4", + "rustix 1.1.5", "slab", "windows-sys 0.61.2", ] @@ -781,7 +781,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", "tokio-websockets", "tracing", @@ -873,7 +873,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -981,9 +981,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -1338,9 +1338,9 @@ dependencies = [ "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -1348,7 +1348,7 @@ dependencies = [ "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tower 0.5.3", "tracing", ] @@ -1532,7 +1532,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -1629,7 +1629,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object 0.37.3", "rustc-demangle", "windows-link 0.2.1", @@ -1724,7 +1724,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1744,7 +1744,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1787,9 +1787,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -1824,6 +1824,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1834,7 +1843,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] @@ -1856,6 +1865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-modes" version = "0.8.1" @@ -1908,7 +1926,7 @@ dependencies = [ "hex", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1940,25 +1958,25 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" dependencies = [ "darling 0.24.1", "ident_case", "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -1982,7 +2000,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -2077,9 +2095,9 @@ dependencies = [ [[package]] name = "byte-unit" -version = "5.2.5" +version = "5.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a813de7f2bbedb7dce265b64f1cf5908ebe4d56281ece8d847e98113788b9b0" +checksum = "c719d56f7e96194cfc53460976d3ba51c85719747c9c62ed99981847b551152b" dependencies = [ "rust_decimal", "schemars 1.2.2", @@ -2120,13 +2138,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -2146,9 +2164,9 @@ dependencies = [ [[package]] name = "bytes-str" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "577d2bf5650f8554d5a372af5ac93535110a0fc75b3e702bb853369febf227c2" +checksum = "4dde6d05e75a31ec9610eb6446a6f0a10dd30ff5100d720fee4c7c7a9008b5ba" dependencies = [ "bytes", "serde", @@ -2293,9 +2311,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" dependencies = [ "find-msvc-tools", "jobserver", @@ -2320,9 +2338,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -2332,12 +2350,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -2390,7 +2408,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -2407,9 +2425,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -2417,9 +2435,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -2429,21 +2447,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cmake" @@ -2454,6 +2472,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cms" version = "0.2.3" @@ -2676,9 +2700,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -2709,9 +2733,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -2739,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2758,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -2818,6 +2842,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2848,6 +2881,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" version = "0.4.90+curl-8.21.0" @@ -3005,7 +3047,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -3060,7 +3102,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -3205,7 +3247,7 @@ dependencies = [ "base64 0.22.1", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.2", "libc", "log", "object_store", @@ -3384,7 +3426,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.14.0", + "indexmap 2.14.2", "paste", "recursive", "serde_json", @@ -3399,7 +3441,7 @@ checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.14.0", + "indexmap 2.14.2", "itertools 0.14.0", "paste", ] @@ -3413,7 +3455,7 @@ dependencies = [ "arrow", "arrow-buffer", "base64 0.22.1", - "blake2", + "blake2 0.10.6", "blake3", "chrono", "datafusion-common", @@ -3553,7 +3595,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "indexmap 2.14.0", + "indexmap 2.14.2", "itertools 0.14.0", "log", "recursive", @@ -3576,7 +3618,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.2", "itertools 0.14.0", "log", "paste", @@ -3638,7 +3680,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.2", "itertools 0.14.0", "log", "parking_lot", @@ -3680,7 +3722,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap 2.14.0", + "indexmap 2.14.2", "log", "recursive", "regex", @@ -3689,9 +3731,9 @@ dependencies = [ [[package]] name = "datasketches" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" [[package]] name = "debug-helper" @@ -3811,7 +3853,7 @@ dependencies = [ "deno_path_util", "deno_unsync", "futures", - "indexmap 2.14.0", + "indexmap 2.14.2", "libc", "parking_lot", "percent-encoding", @@ -3953,8 +3995,8 @@ dependencies = [ "hickory-resolver", "http 1.5.0", "http-body-util", - "hyper 1.11.0", - "hyper-rustls 0.27.9", + "hyper 1.11.1", + "hyper-rustls 0.27.10", "hyper-util", "ipnet", "percent-encoding", @@ -3963,7 +4005,7 @@ dependencies = [ "serde_json", "thiserror 2.0.20", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-socks", "tokio-util", "tokio-vsock", @@ -4080,7 +4122,7 @@ version = "0.228.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "proc-macro-rules", "proc-macro2", "quote", @@ -4156,8 +4198,8 @@ dependencies = [ "deno_error 0.6.1", "deno_tls", "http-body-util", - "hyper 1.11.0", - "hyper-rustls 0.27.9", + "hyper 1.11.1", + "hyper-rustls 0.27.10", "hyper-util", "log", "once_cell", @@ -4455,10 +4497,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -4527,7 +4580,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.2", + "redox_users 0.5.3", "windows-sys 0.61.2", ] @@ -4550,7 +4603,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -4797,7 +4850,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -5002,9 +5055,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" [[package]] name = "fixedbitset" @@ -5024,19 +5077,19 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "rustc_version 0.4.1", ] [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -5122,6 +5175,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "frostem" +version = "1.20260821.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d51501290db793146d5005edc29d3feac221da5702a4c63436344841ea88d4" + [[package]] name = "fs3" version = "0.5.0" @@ -5139,7 +5198,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.59.0", ] @@ -5260,7 +5319,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -5438,9 +5497,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" dependencies = [ "cc", "cfg-if", @@ -5696,9 +5755,9 @@ dependencies = [ [[package]] name = "gosyn" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9758a950dc61a15bc65162f72f5bec7e8efde91f102dc7dce7317f67019a6ba9" +checksum = "28f4ed3cbcb66ddad22553bb3b94087573cf492a34501bc025a23167ed57e856" dependencies = [ "anyhow", "strum", @@ -5738,7 +5797,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.14.0", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -5757,7 +5816,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -5839,7 +5898,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "proc-macro2", "quote", "syn 2.0.119", @@ -5892,9 +5951,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -6109,7 +6168,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -6133,6 +6192,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -6159,9 +6227,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -6189,12 +6257,12 @@ dependencies = [ "futures-util", "headers", "http 1.5.0", - "hyper 1.11.0", - "hyper-rustls 0.27.9", + "hyper 1.11.1", + "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tower-service", ] @@ -6205,7 +6273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6236,7 +6304,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.22.4", @@ -6249,18 +6317,18 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.23.35", "rustls-native-certs 0.8.4", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tower-service", "webpki-roots 1.0.9", ] @@ -6271,7 +6339,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6286,7 +6354,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "native-tls", "tokio", @@ -6301,7 +6369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6321,7 +6389,7 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.11.0", + "hyper 1.11.1", "ipnet", "libc", "percent-encoding", @@ -6342,7 +6410,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6395,11 +6463,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -6449,6 +6538,8 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -6456,6 +6547,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + [[package]] name = "ident_case" version = "1.0.1" @@ -6502,9 +6615,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -6572,11 +6685,11 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -6596,9 +6709,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -6677,9 +6790,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ "defmt", "jiff-core", @@ -6694,18 +6807,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ "jiff-core", "proc-macro2", @@ -6789,9 +6903,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -6835,13 +6949,14 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64 0.21.7", - "pem 1.1.1", - "ring 0.16.20", + "base64 0.22.1", + "js-sys", + "pem", + "ring 0.17.14", "serde", "serde_json", "simple_asn1", @@ -6849,17 +6964,20 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.3.1" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ + "aws-lc-rs", "base64 0.22.1", + "getrandom 0.2.17", "js-sys", - "pem 3.0.6", - "ring 0.17.14", + "pem", "serde", "serde_json", + "signature", "simple_asn1", + "zeroize", ] [[package]] @@ -6890,7 +7008,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", ] [[package]] @@ -6946,15 +7064,15 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-http-proxy", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-timeout", "hyper-util", "jsonpath-rust", "k8s-openapi", "kube-core", - "pem 3.0.6", + "pem", "rustls 0.23.35", "secrecy", "serde", @@ -7149,7 +7267,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "lazy_static", "libgssapi-sys", @@ -7194,14 +7312,14 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.20" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "plain", - "redox_syscall 0.9.3", + "redox_syscall 0.9.4", ] [[package]] @@ -7319,27 +7437,18 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.4" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "lru" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] [[package]] name = "lru-slab" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "lscolors" @@ -7361,9 +7470,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "lzma-sys" @@ -7457,7 +7566,7 @@ dependencies = [ "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "webpki-roots 0.26.11", ] @@ -7682,9 +7791,9 @@ dependencies = [ [[package]] name = "minicov" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" dependencies = [ "cc", "walkdir", @@ -7701,6 +7810,15 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -7708,9 +7826,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -7778,9 +7896,9 @@ dependencies = [ [[package]] name = "murmurhash32" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" +checksum = "a8afc6df942f4c022d70c5725e18df1a705773870e5b7f96fde94ca0334ce77a" [[package]] name = "mysql-common-derive" @@ -7802,10 +7920,11 @@ dependencies = [ [[package]] name = "mysql_async" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2" +checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4" dependencies = [ + "arc-swap", "bytes", "crossbeam-queue", "crossbeam-utils", @@ -7814,12 +7933,12 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.2", + "lru 0.18.4", "mysql_common", "native-tls", - "pem 3.0.6", + "pem", "percent-encoding", - "rand 0.10.2", + "rand 0.10.3", "serde", "socket2 0.6.5", "thiserror 2.0.20", @@ -7837,7 +7956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "btoi", "byteorder", "bytes", @@ -7896,7 +8015,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7907,7 +8026,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -7919,7 +8038,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -7931,7 +8050,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8056,7 +8175,7 @@ dependencies = [ "dirs-sys 0.4.1", "fancy-regex 0.14.0", "heck", - "indexmap 2.14.0", + "indexmap 2.14.2", "log", "lru 0.12.5", "miette", @@ -8234,7 +8353,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "libc", ] @@ -8319,7 +8438,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -8398,7 +8517,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "once_cell", "onig_sys", @@ -8463,7 +8582,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -8771,16 +8890,16 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "stable_deref_trait", ] [[package]] name = "owo-colors" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" [[package]] name = "p12-keystore" @@ -8918,13 +9037,12 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "getrandom 0.4.3", + "phc", ] [[package]] @@ -8961,15 +9079,6 @@ dependencies = [ "hmac", ] -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "pem" version = "3.0.6" @@ -9009,9 +9118,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" dependencies = [ "memchr", "ucd-trie", @@ -9019,9 +9128,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" dependencies = [ "pest", "pest_generator", @@ -9029,9 +9138,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" dependencies = [ "pest", "pest_meta", @@ -9042,9 +9151,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" dependencies = [ "pest", ] @@ -9056,7 +9165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.14.0", + "indexmap 2.14.2", ] [[package]] @@ -9068,6 +9177,17 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", + "getrandom 0.4.3", +] + [[package]] name = "phf" version = "0.11.3" @@ -9311,9 +9431,9 @@ checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "pin-project-lite", - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.61.2", ] @@ -9337,9 +9457,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -9406,6 +9526,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -9447,7 +9569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -9465,7 +9587,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] @@ -9541,7 +9663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3ef4f2f0422f23a82ec9f628ea2acd12871c81a9362b02c43c1aa86acfc3ba1" dependencies = [ "futures", - "indexmap 2.14.0", + "indexmap 2.14.2", "nix 0.30.1", "tokio", "tracing", @@ -9554,7 +9676,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "flate2", "hex", @@ -9568,7 +9690,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", ] @@ -9655,7 +9777,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "getopts", "memchr", "unicase", @@ -9724,9 +9846,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -9744,15 +9866,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", @@ -9843,9 +9965,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -9949,7 +10071,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -9998,7 +10120,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ - "pem 3.0.6", + "pem", "ring 0.17.14", "rustls-pki-types", "time", @@ -10083,16 +10205,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] name = "redox_syscall" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d678d17679829e73d371e96880897e98fee2ded7acc0a50bdf8af2affa4b2fe5" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10108,11 +10230,10 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ - "getrandom 0.2.17", "libredox", "thiserror 2.0.20", ] @@ -10134,7 +10255,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -10212,8 +10333,8 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", - "hyper-rustls 0.27.9", + "hyper 1.11.1", + "hyper-rustls 0.27.10", "hyper-tls", "hyper-util", "js-sys", @@ -10232,7 +10353,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", "tower 0.5.3", "tower-http", @@ -10247,11 +10368,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "encoding_rs", "futures-core", @@ -10260,8 +10381,8 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", - "hyper-rustls 0.27.9", + "hyper 1.11.1", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", @@ -10278,7 +10399,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", "tower 0.5.3", "tower-http", @@ -10299,7 +10420,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.5.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "thiserror 2.0.20", "tower-service", @@ -10316,8 +10437,8 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.5.0", - "hyper 1.11.0", - "reqwest 0.13.4", + "hyper 1.11.1", + "reqwest 0.13.5", "reqwest-middleware", "retry-policies", "thiserror 2.0.20", @@ -10338,7 +10459,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc05fbf560421a0357a750cbe78c7ca19d4923918490daabba313d5dbc871e47" dependencies = [ - "rand 0.10.2", + "rand 0.10.3", ] [[package]] @@ -10426,8 +10547,8 @@ dependencies = [ "oauth2", "pastey", "pin-project-lite", - "rand 0.10.2", - "reqwest 0.13.4", + "rand 0.10.3", + "reqwest 0.13.5", "rmcp-macros", "schemars 1.2.2", "serde", @@ -10445,15 +10566,15 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.1.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d45508e9b4ba024fe996c2638799635d75b6dd0ba8f32ccf08f8026f0c780" +checksum = "cd740c45d66ceb87e5579082abc27bd771665e464e9660a17a048c721b2a6025" dependencies = [ "darling 0.24.1", "proc-macro2", "quote", "serde_json", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -10487,7 +10608,7 @@ dependencies = [ "convert_case 0.10.0", "fnv", "ident_case", - "indexmap 2.14.0", + "indexmap 2.14.2", "proc-macro-crate", "proc-macro2", "quote", @@ -10579,16 +10700,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rust_decimal" version = "1.42.1" @@ -10672,7 +10783,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10681,11 +10792,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", @@ -11100,7 +11211,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.30.0", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -11172,7 +11283,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -11185,7 +11296,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11299,7 +11410,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -11321,7 +11432,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -11330,7 +11441,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -11375,7 +11486,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -11415,16 +11526,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.2", "jiff", "schemars 0.9.0", "schemars 1.2.2", @@ -11436,14 +11547,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -11452,7 +11563,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "ryu", "serde", @@ -11465,7 +11576,7 @@ version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "libyml", "memchr", @@ -11685,9 +11796,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" dependencies = [ "serde", ] @@ -11885,7 +11996,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.14.0", + "indexmap 2.14.2", "log", "memchr", "once_cell", @@ -11948,7 +12059,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "bytes", "chrono", @@ -11992,7 +12103,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "chrono", "crc", @@ -12051,9 +12162,9 @@ dependencies = [ [[package]] name = "sse-stream" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" +checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4" dependencies = [ "bytes", "futures-util", @@ -12242,7 +12353,7 @@ checksum = "72e90b52ee734ded867104612218101722ad87ff4cf74fe30383bd244a533f97" dependencies = [ "anyhow", "bytes-str", - "indexmap 2.14.0", + "indexmap 2.14.2", "serde", "serde_json", "swc_config_macro", @@ -12266,7 +12377,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "is-macro", "num-bigint", "once_cell", @@ -12322,7 +12433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "either", "num-bigint", "phf 0.11.3", @@ -12376,7 +12487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c6f1b8f4232e7a7f614ff7c0f6ccb89c2d028cdf7629f79ad710cff5b28b62c" dependencies = [ "better_scoped_tls", - "indexmap 2.14.0", + "indexmap 2.14.2", "once_cell", "par-core", "phf 0.11.3", @@ -12442,7 +12553,7 @@ checksum = "69ea0052ac23b5b9fbc85bbdb1791b36b918f9d55f594b0ed8e25babb4c32d16" dependencies = [ "base64 0.22.1", "bytes-str", - "indexmap 2.14.0", + "indexmap 2.14.2", "once_cell", "rustc-hash 2.1.3", "serde", @@ -12482,7 +12593,7 @@ version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83259addd99ed4022aa9fc4d39428c008d3d42533769e1a005529da18cde4568" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "num_cpus", "once_cell", "par-core", @@ -12591,9 +12702,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -12620,6 +12731,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "sys-locale" version = "0.3.2" @@ -12655,7 +12777,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "enum-as-inner", "libc", @@ -12683,7 +12805,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -12722,12 +12844,12 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.26.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.27.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "aho-corasick", "arc-swap", - "base64 0.22.1", + "base64 0.23.1", "bitpacking", "bon", "byteorder", @@ -12738,20 +12860,20 @@ dependencies = [ "downcast-rs", "fastdivide", "fnv", + "frostem", "fs4", "htmlescape", "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.16.4", - "lz4_flex 0.13.1", + "lru 0.18.4", + "lz4_flex 0.14.0", "measure_time", "memmap2", "once_cell", "oneshot", "rayon", "regex", - "rust-stemmers", "rustc-hash 2.1.3", "serde", "serde_json", @@ -12768,22 +12890,23 @@ dependencies = [ "thiserror 2.0.20", "time", "typetag", + "unwrap-infallible", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.10.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "downcast-rs", "fastdivide", @@ -12797,8 +12920,8 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.10.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.11.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "async-trait", "byteorder", @@ -12820,8 +12943,8 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.25.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.26.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "fnv", "nom", @@ -12832,8 +12955,8 @@ dependencies = [ [[package]] name = "tantivy-sstable" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "futures-util", "itertools 0.14.0", @@ -12845,8 +12968,8 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "murmurhash32", "tantivy-common", @@ -12854,8 +12977,8 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "serde", ] @@ -12907,7 +13030,7 @@ dependencies = [ "fastrand 2.5.0", "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.61.2", ] @@ -12926,7 +13049,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.61.2", ] @@ -12941,11 +13064,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.2" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" dependencies = [ - "unicode-linebreak", + "icu_segmenter", "unicode-width 0.2.2", ] @@ -12986,7 +13109,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -13127,23 +13250,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tinyvector" @@ -13331,9 +13446,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls 0.23.35", "tokio", @@ -13440,7 +13555,7 @@ dependencies = [ "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-util", ] @@ -13480,7 +13595,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -13489,11 +13604,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -13524,7 +13639,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13533,7 +13648,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -13552,11 +13667,12 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", + "flate2", "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13565,12 +13681,13 @@ dependencies = [ "rustls-native-certs 0.8.4", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-stream", "tower 0.5.3", "tower-layer", "tower-service", "tracing", + "zstd", ] [[package]] @@ -13601,7 +13718,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.0", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", @@ -13636,7 +13753,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -13816,9 +13933,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" +checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" [[package]] name = "tree-sitter-r" @@ -13909,9 +14026,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -13952,7 +14069,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -14050,21 +14167,15 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-id-start" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" +checksum = "0825d2e09dcae814a4273c1e1435979d414ede25b2ee235b1545da6682483765" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-normalization" @@ -14148,7 +14259,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -14176,6 +14287,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap-infallible" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e497bb1f828cc9fb236722c2eaa100dcf201563f38f4da6252357a59037adf31" + [[package]] name = "ureq" version = "2.12.1" @@ -14259,9 +14376,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14276,11 +14393,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "fslock", "gzip-header", "home", - "miniz_oxide", + "miniz_oxide 0.8.9", "paste", "which 6.0.3", ] @@ -14382,9 +14499,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -14396,9 +14513,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -14406,9 +14523,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -14416,31 +14533,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895a2607575412a4eda1df892084a375ea10dfeadc4d7d2ab87b854e4ddc7ba1" +checksum = "45863ef0bef521c12124eb39d9a38513c47db75f22e503c06beacd40afeb35db" dependencies = [ "async-trait", "cast", @@ -14460,20 +14577,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288cb0ebe215033bf949ae1fd046726daa4c32a157f24b9dc6ac387a52aa759" +checksum = "8c89dcab8b516b6b603baca9d550b7282d68fcc7f367e3956cff7ebf406a3f12" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] name = "wasm-bindgen-test-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff1c1b360982e93b6d8ea9c04836f71dba0817a16f91e229cf3a51bdd9d987" +checksum = "f37b4f992cebe528ef34964ae69681ac0fe7080071e7298e46008f9d380302af" [[package]] name = "wasm-streams" @@ -14527,9 +14644,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -14671,7 +14788,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-nats", @@ -14688,6 +14805,7 @@ dependencies = [ "git-version", "hex", "hmac", + "jsonwebtoken 10.4.0", "lazy_static", "once_cell", "opentelemetry 0.30.0", @@ -14695,7 +14813,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rumqttc", "rustls 0.23.35", "serde", @@ -14715,6 +14833,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", + "tower-cookies", "tracing", "tracing-subscriber", "url", @@ -14726,6 +14845,7 @@ dependencies = [ "windmill-api-client", "windmill-api-scripts", "windmill-api-settings", + "windmill-api-users", "windmill-autoscaling", "windmill-common", "windmill-dep-map", @@ -14756,7 +14876,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.798.1" +version = "1.816.0" dependencies = [ "async-stream", "async-trait", @@ -14770,9 +14890,10 @@ dependencies = [ "eventsource-stream", "futures", "http 1.5.0", + "indexmap 2.14.2", "lazy_static", "mime_guess", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -14789,7 +14910,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14802,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "argon2", @@ -14832,10 +14953,10 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", - "indexmap 2.14.0", + "hyper 1.11.1", + "indexmap 2.14.2", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "libxml", "magic-crypt", @@ -14852,7 +14973,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rsa", "rust-embed", "rustls 0.23.35", @@ -14942,12 +15063,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "quick_cache", "serde", @@ -14965,7 +15086,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14982,17 +15103,17 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "axum 0.8.9", "chrono", "http 1.5.0", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15008,7 +15129,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.798.1" +version = "1.816.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15018,7 +15139,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15035,7 +15156,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15057,7 +15178,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15066,7 +15187,7 @@ dependencies = [ "candle-transformers", "hf-hub", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -15080,7 +15201,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15096,11 +15217,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", - "hyper 1.11.0", + "hyper 1.11.1", "serde", "serde_json", "sql-builder", @@ -15118,7 +15239,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15129,6 +15250,7 @@ dependencies = [ "serde_json", "sql-builder", "sqlx", + "tracing", "uuid", "windmill-api-auth", "windmill-api-workspaces", @@ -15139,7 +15261,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15153,7 +15275,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-nats", @@ -15167,7 +15289,7 @@ dependencies = [ "hmac", "rand 0.9.0", "rdkafka", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "rumqttc", "serde", @@ -15188,14 +15310,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "axum 0.8.9", "base64 0.22.1", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15213,7 +15335,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15222,7 +15344,7 @@ dependencies = [ "hex", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "semver 1.0.28", "serde", "serde_json", @@ -15241,12 +15363,12 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "axum 0.8.9", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.2", "itertools 0.14.0", "lazy_static", "serde", @@ -15263,7 +15385,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15283,18 +15405,18 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "prometheus", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -15321,7 +15443,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15343,13 +15465,14 @@ dependencies = [ "windmill-ai", "windmill-alerting", "windmill-api-auth", + "windmill-audit", "windmill-common", "windmill-object-store", ] [[package]] name = "windmill-api-sse" -version = "1.798.1" +version = "1.816.0" dependencies = [ "lazy_static", "serde", @@ -15361,13 +15484,13 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.798.1" +version = "1.816.0" dependencies = [ "argon2", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15385,7 +15508,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15399,14 +15522,14 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.798.1" +version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "regex", @@ -15434,7 +15557,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.798.1" +version = "1.816.0" dependencies = [ "chrono", "lazy_static", @@ -15448,7 +15571,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15467,7 +15590,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.798.1" +version = "1.816.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15484,7 +15607,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "chrono-tz", @@ -15504,10 +15627,10 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.11.0", - "indexmap 2.14.0", + "hyper 1.11.1", + "indexmap 2.14.2", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "magic-crypt", "mail-send", @@ -15523,12 +15646,13 @@ dependencies = [ "pep440_rs", "phf 0.11.3", "pin-project-lite", + "pkcs1", "postgres-native-tls 0.5.3", "prometheus", "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "reqwest-retry", "rsa", @@ -15537,8 +15661,10 @@ dependencies = [ "serde", "serde_json", "serde_yml", + "sha1", "sha2 0.10.9", "size", + "spki", "sqlx", "strum", "strum_macros", @@ -15571,9 +15697,10 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.798.1" +version = "1.816.0" dependencies = [ "chrono", + "futures", "itertools 0.14.0", "lazy_static", "serde", @@ -15590,7 +15717,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.798.1" +version = "1.816.0" dependencies = [ "regex", "serde", @@ -15599,22 +15726,26 @@ dependencies = [ "tokio", "tracing", "uuid", + "windmill-audit", "windmill-common", + "windmill-dep-map", "windmill-queue", ] [[package]] name = "windmill-indexer" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "astral-tokio-tar", "bytes", "chrono", "const_format", + "datafusion", "flume", "futures", "lazy_static", + "object_store", "serde", "serde_json", "sqlx", @@ -15622,6 +15753,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "url", "uuid", "windmill-common", "windmill-object-store", @@ -15629,7 +15761,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "futures", @@ -15646,7 +15778,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.798.1" +version = "1.816.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15662,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -15670,7 +15802,7 @@ dependencies = [ "futures", "http 1.5.0", "oauth2", - "reqwest 0.13.4", + "reqwest 0.13.5", "rmcp", "serde", "serde_json", @@ -15683,7 +15815,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -15695,7 +15827,7 @@ dependencies = [ "http 1.5.0", "itertools 0.14.0", "lazy_static", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -15714,7 +15846,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "arc-swap", @@ -15739,7 +15871,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-stream", @@ -15757,9 +15889,10 @@ dependencies = [ "lazy_static", "object_store", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", + "serial_test", "sqlx", "tempfile", "tokio", @@ -15773,7 +15906,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "futures", @@ -15791,7 +15924,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.798.1" +version = "1.816.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15800,7 +15933,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -15812,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde_json", @@ -15824,7 +15957,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "gosyn", @@ -15836,7 +15969,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -15848,7 +15981,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde_json", @@ -15860,7 +15993,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "nu-parser", @@ -15871,7 +16004,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15882,7 +16015,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15894,7 +16027,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15905,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-recursion", @@ -15927,7 +16060,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde_json", @@ -15939,7 +16072,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -15953,7 +16086,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15970,7 +16103,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -15983,7 +16116,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde", @@ -15995,7 +16128,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -16013,7 +16146,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16029,7 +16162,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16045,7 +16178,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -16059,7 +16192,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-recursion", @@ -16080,7 +16213,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "serde_urlencoded", @@ -16098,7 +16231,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "const_format", @@ -16122,7 +16255,7 @@ dependencies = [ "lazy_static", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "rustls 0.23.35", "serde", "serde_json", @@ -16138,7 +16271,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.798.1" +version = "1.816.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16149,7 +16282,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-recursion", @@ -16160,11 +16293,11 @@ dependencies = [ "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16184,7 +16317,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16208,18 +16341,18 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sql-builder", @@ -16241,7 +16374,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16268,7 +16401,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16283,7 +16416,7 @@ dependencies = [ "lazy_static", "quick_cache", "rand 0.9.0", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -16301,7 +16434,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16321,7 +16454,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16333,10 +16466,10 @@ dependencies = [ "google-cloud-pubsub", "http 1.5.0", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "quick_cache", - "reqwest 0.13.4", + "reqwest 0.13.5", "serde", "serde_json", "sqlx", @@ -16355,7 +16488,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16367,7 +16500,7 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -16391,7 +16524,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16414,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16438,7 +16571,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-nats", @@ -16462,7 +16595,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16497,7 +16630,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16525,7 +16658,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-trait", @@ -16550,10 +16683,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", "itertools 0.14.0", @@ -16569,7 +16702,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-once-cell", @@ -16598,10 +16731,10 @@ dependencies = [ "hmac", "hudsucker", "hyper-http-proxy", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "libffi-sys", "libloading", @@ -16614,7 +16747,8 @@ dependencies = [ "opentelemetry 0.30.0", "opentelemetry-proto 0.30.0", "oracle", - "pem 3.0.6", + "parquet", + "pem", "pep440_rs", "postgres-native-tls 0.5.3", "process-wrap", @@ -16624,7 +16758,7 @@ dependencies = [ "rand 0.9.0", "rcgen", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "reqwest-middleware", "rsa", "rust_decimal", @@ -16641,7 +16775,7 @@ dependencies = [ "tiberius", "tokio", "tokio-postgres", - "tokio-rustls 0.26.4", + "tokio-rustls 0.26.5", "tokio-stream", "tokio-util", "tracing", @@ -16686,7 +16820,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.798.1" +version = "1.816.0" dependencies = [ "bytes", "futures", @@ -16937,7 +17071,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "widestring", "windows-sys 0.52.0", ] @@ -17319,7 +17453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.4", + "rustix 1.1.5", ] [[package]] @@ -17374,30 +17508,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", @@ -17415,14 +17549,14 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] @@ -17454,6 +17588,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -17462,6 +17597,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -17475,7 +17611,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -17485,16 +17621,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", - "indexmap 2.14.0", + "indexmap 2.14.2", "memchr", "typed-path", ] [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] name = "zmij" @@ -17513,18 +17649,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 75cc6bc007..b25f331b88 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.798.1" +version = "1.816.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.798.1" +version = "1.816.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -108,6 +108,10 @@ debug = "line-tables-only" [profile.dev.package."*"] debug = false +# The expression parser's fallback call chain exhausts worker stacks without optimization. +[profile.dev.package.php-parser-rs] +opt-level = 1 + [profile.release] lto = "thin" debug = "line-tables-only" @@ -115,7 +119,7 @@ strip = "none" [features] default = [] -private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] +private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer?/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"] local_reports = ["windmill-common/local_reports"] @@ -131,6 +135,10 @@ quickjs = ["windmill-worker/quickjs", "windmill-api/quickjs"] openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect", "windmill-object-store/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] +# `tantivy` must stay the only feature that enables windmill-indexer: it is the only one that +# also gives it `enterprise` + `parquet`, and its EE sources gate nearly everything on that +# pair, so a bare windmill-indexer is a crate of dead code that `-D warnings` rejects. Any +# other feature wanting one of its features has to use the optional `windmill-indexer?/` form. tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:windmill-runtime-nativets", "windmill-test-utils/deno_core"] @@ -347,6 +355,8 @@ windmill-trigger-sqs.workspace = true windmill-trigger-gcp.workspace = true windmill-trigger-azure.workspace = true windmill-api-auth.workspace = true +tower-cookies.workspace = true +windmill-api-users.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true @@ -361,6 +371,7 @@ aws-config.workspace = true aws-credential-types.workspace = true hmac.workspace = true hex.workspace = true +jsonwebtoken = { workspace = true } [workspace.dependencies] @@ -477,7 +488,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] } mime_guess = "^2" hex = "^0" sql-builder = "^3" -argon2 = "^0" +# Minor-pinned rather than the `^0` used elsewhere in this file: argon2's 0.x +# minors are API-breaking (0.6 moved `SaltString` into `phc`, put `rand_core` +# behind a feature and changed `hash_password`), so a float breaks the build. +argon2 = "0.6" quick_cache = "^0" rand = "=0.9.0" rand_core = { version = "^0", features = ["std"] } @@ -590,6 +604,8 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } const-str = "0.5" constant_time_eq = "0.3.1" rsa = "^0" +spki = { version = "0.7", features = ["pem"] } +pkcs1 = "0.7" aes-gcm = "0.10.3" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" @@ -621,7 +637,9 @@ libxml = { version = "=0.3.3" } samael = { git="https://github.com/njaremko/samael", rev="f879f1942ec1b34b6d3027ce7e4724ad95d15dfa", features = ["xmlsec"] } gcp_auth = "0.9.0" rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} -jsonwebtoken = "8.3.0" +# aws_lc_rs, not rust_crypto: the RustCrypto backend refuses RSA keys above 4096 bits, and +# external/guest JWT issuers may sign with 8192-bit keys. +jsonwebtoken = { version = "^10.3", features = ["aws_lc_rs"] } pem = "3.0.1" nix = { version = "0.27.1", features = ["process", "signal"] } fs4 = "0.13" @@ -653,6 +671,12 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" datafusion = "47.0.0" +# The row API only: a dbt engine's parquet index is six string columns, so this +# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for +# every shipped edition (`oss_core`), and cargo unifies the features there; this +# set is what a build WITHOUT object storage compiles. ZSTD is what the engine +# writes today, snap what parquet writers most often default to. +parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" @@ -679,7 +703,7 @@ async-stream = "^0" opentelemetry = "0.30.0" tracing-opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] } -opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto"] } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto", "gzip-tonic", "zstd-tonic"] } opentelemetry-appender-tracing = "0.30.0" opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] } opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] } @@ -697,7 +721,7 @@ tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "^0" pin-project-lite = "^0" -tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" } +tantivy = { git="https://github.com/windmill-labs/tantivy", rev="ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" } backon = "1.3.0" diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index 0468ebf721..af33877078 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -85,7 +85,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream | | EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation | | EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts | -| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | +| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers, script-controlled `wm_content_type`/`wm_headers` on `run_wait_result` and sync HTTP-route responses | stored user content → admin browser (same origin) | Admin session, account takeover | | EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | | EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | | EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS` now defaults to `true`; can still be overridden to `false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | @@ -104,9 +104,9 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b | | T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j | | T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | -| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | +| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override; release images (`v*` tags) keyless-signed with cosign, with per-platform SPDX SBOMs embedded at build time (covered by the signed index digest) + SLSA provenance (GitHub artifact attestations) | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | -| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | +| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, S3 download content-type, or a script-chosen `text/html` content type on `run_wait_result` / sync HTTP-route responses (GET-reachable with the `SameSite=Lax` session cookie) | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads and on every `result_to_response` composite result (inserted after `wm_headers`; hop-by-hop names such as `Connection` rejected so a proxy cannot strip them) | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0, WIN-2471 | | T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | | T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 | | T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d | @@ -169,4 +169,4 @@ check. | Canonicalize + confine all file-path inputs to a base dir and never follow symlinks in log/file readers | T13 | yes | S | | Mask secrets at the log sink and keep secrets out of worker process env (`/proc`) — pass via files/pipes scrubbed after use | T15 | partial | M | | Add global rate limiting and per-tenant resource/queue quotas at the edge | T16, T18 | partial | M | -| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release | T9 | partial | M | +| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release — release images now cosign-signed with SBOM + SLSA provenance attestations; remaining: CI action SHA-pinning, hub-script integrity, rhel/rpi images | T9 | partial | M | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d09a8a6d56..f43dd16d02 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a9102dfd61e598fa77c157f940ffa69613a2f98f +029ba12d9e08a376a866acc274bd1e0176c6b353 diff --git a/backend/migrations/20260622072905_ai_free_token_usage.down.sql b/backend/migrations/20260622072905_ai_free_token_usage.down.sql new file mode 100644 index 0000000000..95879034e4 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE ai_free_token_daily_usage; +DROP TABLE ai_free_token_usage; diff --git a/backend/migrations/20260622072905_ai_free_token_usage.up.sql b/backend/migrations/20260622072905_ai_free_token_usage.up.sql new file mode 100644 index 0000000000..ae0cf47e45 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.up.sql @@ -0,0 +1,19 @@ +-- One-time grant of the Windmill-provided free AI tier, measured as cost in nano-dollars +-- (1e-9 USD) rather than raw tokens — a prompt-cache hit costs a fraction of a fresh input +-- token, so a token count wildly overstates the real bill. The grant never resets: once +-- spent, the user must bring their own API key. Keyed by normalized email so the allowance +-- is shared across a user's workspaces (and is resistant to +tag / gmail-dot aliasing). +CREATE TABLE ai_free_token_usage ( + email VARCHAR(255) PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Instance-wide daily cost ceiling (nano-dollars) for the free tier — a kill-switch +-- independent of the per-user grant, bounding the blast radius of a bad day. One row per +-- UTC day. +CREATE TABLE ai_free_token_daily_usage ( + day DATE PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql b/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql new file mode 100644 index 0000000000..b21e2bb474 --- /dev/null +++ b/backend/migrations/20260714142042_add_git_sync_ci_test_check.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS git_sync_ci_test_check; diff --git a/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql b/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql new file mode 100644 index 0000000000..59fabb890c --- /dev/null +++ b/backend/migrations/20260714142042_add_git_sync_ci_test_check.up.sql @@ -0,0 +1,39 @@ +-- One "Windmill CI tests" GitHub check run per (fork workspace, repository, PR head commit): +-- the pull_request webhook opens the check in_progress and it is concluded once +-- the fork's CI tests settle, so the results can gate a GitHub PR. +CREATE TABLE git_sync_ci_test_check ( + -- The fork workspace whose CI tests gate the PR: keys the row, and its `ci_test` + -- jobs are what the check reflects. + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + head_sha VARCHAR(64) NOT NULL, + -- The PR's head branch: the check waits until the fork's synced state for this + -- branch (written by its pushes and pulls alike) names `head_sha`. + head_ref VARCHAR(255) NOT NULL, + -- The workspace whose git host credential posts the check: the one that received + -- the pull request webhook (the parent owning the repo hook). + poster_workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + repo_url TEXT NOT NULL, + -- The fork's copy of the repository resource: keys the synced-head lookup, since a + -- fork syncing two repositories names its branch identically in both. + repo_resource_path VARCHAR(255) NOT NULL, + -- NULL when the GitHub check-run creation failed; the poller retries the create. + check_run_id BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + concluded BOOLEAN NOT NULL DEFAULT false, + conclusion TEXT, + concluded_at TIMESTAMPTZ, + -- Decoupled from `concluded` so a failed check-run PATCH is retried by the + -- poller instead of hanging a required check on GitHub. + github_posted BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (workspace_id, repo_resource_path, head_sha) +); + +-- Rows still needing action (create retry, conclusion, timeout, delivery retry). +-- A row drops out only once it is both concluded and delivered to GitHub, so the +-- per-job conclusion hook and the poller sweeper both scan a small live set. +CREATE INDEX idx_git_sync_ci_test_check_pending + ON git_sync_ci_test_check (workspace_id) + WHERE NOT concluded OR NOT github_posted; + +GRANT ALL ON git_sync_ci_test_check TO windmill_user; +GRANT ALL ON git_sync_ci_test_check TO windmill_admin; diff --git a/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql b/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql new file mode 100644 index 0000000000..bee3e6034c --- /dev/null +++ b/backend/migrations/20260825105019_http_trigger_allowed_origins.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE http_trigger DROP COLUMN allowed_origins; diff --git a/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql b/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql new file mode 100644 index 0000000000..444fc46535 --- /dev/null +++ b/backend/migrations/20260825105019_http_trigger_allowed_origins.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE http_trigger ADD COLUMN allowed_origins TEXT[]; diff --git a/backend/migrations/20260827203157_login_link.down.sql b/backend/migrations/20260827203157_login_link.down.sql new file mode 100644 index 0000000000..aa26e1eee8 --- /dev/null +++ b/backend/migrations/20260827203157_login_link.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS login_link; diff --git a/backend/migrations/20260827203157_login_link.up.sql b/backend/migrations/20260827203157_login_link.up.sql new file mode 100644 index 0000000000..0be611f408 --- /dev/null +++ b/backend/migrations/20260827203157_login_link.up.sql @@ -0,0 +1,13 @@ +-- Single-use login links minted by a superadmin for one account. Consumed by an +-- unauthenticated GET that mints a session; the row is never a bearer credential itself. +CREATE TABLE login_link ( + token_hash CHAR(64) PRIMARY KEY, + email VARCHAR(255) NOT NULL REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + rd TEXT, + expiration TIMESTAMPTZ NOT NULL, + consumed_at TIMESTAMPTZ, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX login_link_email_idx ON login_link (email); diff --git a/backend/migrations/20260828190901_cloud_trial_offer.down.sql b/backend/migrations/20260828190901_cloud_trial_offer.down.sql new file mode 100644 index 0000000000..0769b91752 --- /dev/null +++ b/backend/migrations/20260828190901_cloud_trial_offer.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cloud_trial_offer; diff --git a/backend/migrations/20260828190901_cloud_trial_offer.up.sql b/backend/migrations/20260828190901_cloud_trial_offer.up.sql new file mode 100644 index 0000000000..d16ea6857c --- /dev/null +++ b/backend/migrations/20260828190901_cloud_trial_offer.up.sql @@ -0,0 +1,11 @@ +-- A pre-approved self-hosted Enterprise trial offered to an account created through a +-- pre-approved invite. No expiry: the offer lasts until a trial or subscription exists. +-- The cascade follows the account out on deletion and rename. A superadmin users-import +-- replaces every account by deleting and reinserting it, which takes these rows with it: +-- the offers, like the onboarding profiles, are recorded by the portal that minted them. +CREATE TABLE cloud_trial_offer ( + email VARCHAR(255) PRIMARY KEY REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + consumed_at TIMESTAMPTZ, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260830085453_log_file_indexed_at.down.sql b/backend/migrations/20260830085453_log_file_indexed_at.down.sql new file mode 100644 index 0000000000..ac3c92f3eb --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS index_log_file_premigration; +DROP INDEX IF EXISTS index_log_file_pending_path; +DROP INDEX IF EXISTS index_log_file_pending; +ALTER TABLE log_file DROP COLUMN IF EXISTS indexed_at; diff --git a/backend/migrations/20260830085453_log_file_indexed_at.up.sql b/backend/migrations/20260830085453_log_file_indexed_at.up.sql new file mode 100644 index 0000000000..3421a5082a --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.up.sql @@ -0,0 +1,35 @@ +-- The service log ingest walked `log_file` with a cursor over `log_ts`, which is when a line +-- was written rather than when its row appeared. Rows do not arrive in that order — an upload +-- retried after a failure, a host that has just started, a batch the row limit cut mid-minute — +-- and a row that becomes visible behind the cursor is never read: it stays in `log_file` and its +-- lines stay out of search until retention drops them. +-- +-- No ordering fixes this. A cursor over arrival order fails the same way, because `nextval` is +-- allocated before its INSERT commits: a row can be assigned a lower value and commit after a +-- higher one has already moved the cursor past it. Which rows are outstanding is a property of +-- the rows, so it is recorded on them. +ALTER TABLE log_file ADD COLUMN indexed_at TIMESTAMPTZ; + +-- Rows that already existed are marked, not queued: on a 14-day window most were ingested long +-- ago and their raw files are gone. A sentinel rather than a timestamp, because the indexer has to +-- tell them apart from rows registered since — those start NULL — and it puts the window's worth of +-- them back on the queue on its first pass, keeping only what the columnar store can vouch for. +-- +-- Not split here on the cursor the old ingest had reached. Below that cursor sits every row it +-- skipped, which is the loss this migration exists to stop; recording those as done would carry the +-- bug into its own fix. +UPDATE log_file SET indexed_at = 'epoch' WHERE indexed_at IS NULL; + +-- The work queue, and the only index the ingest query needs: outstanding rows are a small +-- fraction of the table, so this stays proportional to what is left to do rather than to the +-- retention window. +CREATE INDEX index_log_file_pending ON log_file (log_ts) WHERE indexed_at IS NULL; + +-- A rebuild takes rows out of the queue by the file it just read out of the store, which is +-- the one lookup that arrives without a `log_ts`: the primary key is `(hostname, log_ts)`, so +-- nothing else covers it and each batch would scan every outstanding row instead. +CREATE INDEX index_log_file_pending_path ON log_file (hostname, file_path) WHERE indexed_at IS NULL; + +-- Reached once per pass while pre-migration rows survive, and never again after the first +-- conversion clears them. +CREATE INDEX index_log_file_premigration ON log_file (log_ts) WHERE indexed_at = 'epoch'; diff --git a/backend/migrations/20260831120907_skills_resource_type.down.sql b/backend/migrations/20260831120907_skills_resource_type.down.sql new file mode 100644 index 0000000000..5cc32da37d --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.down.sql @@ -0,0 +1,13 @@ +-- The up migration only ever added: `ai_skill` still holds every skill it copied, +-- so there is nothing to restore and nothing to delete. Removing the resources +-- would destroy any a user has since edited or created, and removing a folder +-- would take whatever else was put in it. +-- +-- The seeded resource type goes. `created_by` only distinguishes this migration's +-- row from one a user created by hand: a hub sync updates the schema in place and +-- leaves `created_by` alone, so a synced-over row is still removed here and the +-- next sync puts it back. +DELETE FROM resource_type +WHERE workspace_id = 'admins' + AND name = 'ai_skill' + AND created_by = 'system'; diff --git a/backend/migrations/20260831120907_skills_resource_type.up.sql b/backend/migrations/20260831120907_skills_resource_type.up.sql new file mode 100644 index 0000000000..4e8f9cfa95 --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.up.sql @@ -0,0 +1,88 @@ +-- AI chat skills move from the `ai_skill` table onto ordinary resources, so they +-- gain folder ACLs, version history, workspace export and git-sync. An `ai_skill` +-- resource holds the SKILL.md body in `value.content`; its description lives in +-- the resource's own `description` column and its name is the path basename. +-- +-- Nothing here is destructive. `ai_skill` is left in place, unread, for a later +-- release to drop once operators have confirmed the copy. That is what lets every +-- step below skip on conflict rather than resolve one: a skipped row is still in +-- the table, so it is not lost, and the migration needs no record of what it did +-- in order to be reversible. + +-- `format_extension` makes the resource editor render `value.content` as a plain +-- .md file. Seeded under 'admins' so every workspace sees it. +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, format_extension, edited_at) +VALUES ( + 'admins', + 'ai_skill', + '{"type": "object", "properties": {"content": {"type": "string"}}}', + 'A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.', + 'system', + 'md', + now() +) +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Shared home matching the admin-only upload these skills had. A workspace that +-- already has a `skills` folder keeps it untouched, ACL and all: adopting one +-- would hand its own grants — possibly write for everyone — over a set of +-- instructions the assistant follows. +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) +SELECT DISTINCT workspace_id, 'skills', 'Skills', ARRAY[]::TEXT[], '{"g/all": false}'::jsonb +FROM ai_skill +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Copied only where the destination is free and the folder matches the one above +-- exactly, owners included: a pre-existing folder carrying the same ACL but an +-- owner would hand that owner update and delete over skills the removed API let +-- only workspace admins touch. Anything else stays in `ai_skill` for an operator +-- to place deliberately. +-- +-- What was actually inserted is recorded rather than inferred. Inferring it from +-- "is there an ai_skill resource at the destination" reports nothing when the +-- blocker is itself an ai_skill with different instructions — the one case where +-- the skipped skill is least likely to be noticed. +CREATE TEMP TABLE ai_skill_copied AS +WITH inserted AS ( + INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) + SELECT + s.workspace_id, + 'f/skills/' || s.name, + jsonb_build_object('content', s.instructions), + s.description, + 'ai_skill', + s.edited_by, + s.edited_at + FROM ai_skill s + JOIN folder f + ON f.workspace_id = s.workspace_id + AND f.name = 'skills' + AND f.extra_perms = '{"g/all": false}'::jsonb + AND cardinality(f.owners) = 0 + ON CONFLICT (workspace_id, path) DO NOTHING + RETURNING workspace_id, path +) +SELECT workspace_id, path FROM inserted; + +-- Anything not copied is still in `ai_skill`, but nothing reads that table any +-- more, so from the app's side the skill is missing until an operator places it. +-- Name them rather than leaving that to be discovered. +DO $$ +DECLARE + leftover RECORD; +BEGIN + FOR leftover IN + SELECT s.workspace_id, s.name + FROM ai_skill s + WHERE NOT EXISTS ( + SELECT 1 FROM ai_skill_copied c + WHERE c.workspace_id = s.workspace_id + AND c.path = 'f/skills/' || s.name + ) + LOOP + RAISE WARNING 'ai_skill %/% was not copied to a resource (its destination or the f/skills folder is already taken); it remains in the ai_skill table', + leftover.workspace_id, leftover.name; + END LOOP; +END $$; + +DROP TABLE ai_skill_copied; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql new file mode 100644 index 0000000000..e62fc02975 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql @@ -0,0 +1,12 @@ +-- Drafts owned by a principal with no login account cannot exist under the constraint; drop them +-- before restoring it. +DELETE FROM draft +WHERE email IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM password WHERE password.email = draft.email); + +ALTER TABLE draft + ADD CONSTRAINT draft_password_fkey + FOREIGN KEY (email) + REFERENCES password(email) + ON DELETE CASCADE + ON UPDATE CASCADE; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql new file mode 100644 index 0000000000..65a6686e95 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql @@ -0,0 +1,3 @@ +-- The delete and rename this cascaded are now explicit, at the sites that remove or rename an +-- account; `windmill_common::user_drafts::delete_drafts_of_email` carries the reasoning. +ALTER TABLE draft DROP CONSTRAINT IF EXISTS draft_password_fkey; diff --git a/backend/migrations/20260901192755_guest_app_access.down.sql b/backend/migrations/20260901192755_guest_app_access.down.sql new file mode 100644 index 0000000000..065e60c0c8 --- /dev/null +++ b/backend/migrations/20260901192755_guest_app_access.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS guest_activity; +ALTER TABLE workspace_settings DROP COLUMN guest_access_enabled; diff --git a/backend/migrations/20260901192755_guest_app_access.up.sql b/backend/migrations/20260901192755_guest_app_access.up.sql new file mode 100644 index 0000000000..a29582ba99 --- /dev/null +++ b/backend/migrations/20260901192755_guest_app_access.up.sql @@ -0,0 +1,28 @@ +-- Guest app access: a workspace-level switch, off by default. An app whose policy says +-- `execution_mode: guest` only admits guests where this is on, and the check runs where +-- the guest session is minted -- an app definition carries its policy, so git-sync and +-- the CLI push `guest` past every UI gate. +ALTER TABLE workspace_settings + ADD COLUMN guest_access_enabled BOOLEAN NOT NULL DEFAULT false; + +-- A guest leaves no `usr` or `password` row, which is what keeps them off every seat +-- counter, so this is the only durable record that one was here: a row per guest, +-- workspace and day, written when the session is minted. +-- +-- Deliberately not the audit log. The seat scan is served by a partial index whose +-- predicate names the login operations literally, and `audit_partitioned` is a +-- partitioned table, where `CREATE INDEX CONCURRENTLY` is unsupported -- adding a +-- guest operation to that predicate means a locking rebuild on the largest table an +-- instance has. Guest logins still write `users.login_guest` for the audit trail; +-- nothing counts them from there. +CREATE TABLE guest_activity ( + email VARCHAR(255) NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + day DATE NOT NULL DEFAULT CURRENT_DATE, + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (email, workspace_id, day) +); + +-- The retention delete filters on day alone; the PK only reaches it through two +-- other columns. +CREATE INDEX idx_guest_activity_day ON guest_activity (day); diff --git a/backend/migrations/20260903071242_guest_jwt_entry.down.sql b/backend/migrations/20260903071242_guest_jwt_entry.down.sql new file mode 100644 index 0000000000..f3e758819b --- /dev/null +++ b/backend/migrations/20260903071242_guest_jwt_entry.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE guest_activity DROP COLUMN jwt_entry; +ALTER TABLE workspace_settings + DROP CONSTRAINT workspace_settings_guest_jwt_one_key, + DROP COLUMN guest_jwt_public_key, + DROP COLUMN guest_jwt_jwks_url; diff --git a/backend/migrations/20260903071242_guest_jwt_entry.up.sql b/backend/migrations/20260903071242_guest_jwt_entry.up.sql new file mode 100644 index 0000000000..0cc449f539 --- /dev/null +++ b/backend/migrations/20260903071242_guest_jwt_entry.up.sql @@ -0,0 +1,15 @@ +-- A second way in for a guest: a JWT minted by the embedding customer's own backend and +-- verified against a key the workspace admin configured. One key shape per workspace, +-- a PEM public key or a JWKS URL, never both: a token is verified against exactly one +-- source, and two would make "which one refused it" undiagnosable. +ALTER TABLE workspace_settings + ADD COLUMN guest_jwt_public_key TEXT, + ADD COLUMN guest_jwt_jwks_url TEXT, + ADD CONSTRAINT workspace_settings_guest_jwt_one_key + CHECK (guest_jwt_public_key IS NULL OR guest_jwt_jwks_url IS NULL); + +-- Whether the guest came in on a JWT that day (as opposed to, or as well as, an +-- identity-provider sign-in). The seat telemetry reports the two entries apart, since +-- an app-only user routed through a guest JWT is one that `jwt_ext_` would have counted. +ALTER TABLE guest_activity + ADD COLUMN jwt_entry BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql new file mode 100644 index 0000000000..ca88ec0f1e --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS git_credentials; diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql new file mode 100644 index 0000000000..7ece575cc1 --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql @@ -0,0 +1,8 @@ +-- Server-owned git-sync credentials, one entry per repository the workspace +-- holds a token for, keyed by that repository rather than by a resource naming +-- it: a resource's URL is writable, and its path is not settled while it is +-- being created. +-- Kept out of `git_sync` because that column is copied into forks and returned +-- by the workspace settings API; this one is copied by neither. +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS git_credentials JSONB NOT NULL DEFAULT '[]'::jsonb; diff --git a/backend/migrations/20260904135713_dbt_environment_state.down.sql b/backend/migrations/20260904135713_dbt_environment_state.down.sql new file mode 100644 index 0000000000..dab9025417 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS dbt_environment_state; diff --git a/backend/migrations/20260904135713_dbt_environment_state.up.sql b/backend/migrations/20260904135713_dbt_environment_state.up.sql new file mode 100644 index 0000000000..a49df5b3d1 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.up.sql @@ -0,0 +1,52 @@ +-- The dbt state one project last built into one environment: the `manifest.json` +-- (and the `run_results.json` beside it) that `dbt --defer --state ` resolves +-- an unbuilt `ref()` through. +-- +-- Separate from `dbt_run_state`, which answers a different question. That one is +-- keyed by the executing principal and holds the LAST run whatever its outcome, +-- so `dbt retry` can resume its failures; this one is keyed by environment and +-- holds the last SUCCESSFUL run, because a relation a later run defers to has to +-- exist. Merging them would make a retry resume a run that is not the last one, +-- or a deferral point at relations a failed run never wrote. +CREATE TABLE IF NOT EXISTS dbt_environment_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- The workspace warehouse, the dbt target, and the database and schema that + -- target resolves to. All four, because deferring is resolving a relation + -- NAME: a repointed warehouse or a moved schema makes the stored manifest + -- describe relations that are not where this run would look for them, and the + -- run has no other way to notice. A move therefore reads as an environment + -- with no state yet rather than as state that silently no longer fits. + -- + -- TEXT rather than VARCHAR(255): a project bringing its own `profiles.yml` + -- spells its own schema and database, so the length is the project's. + environment TEXT NOT NULL, + -- The run that published it, so a deferring run can say what it deferred to. + job_id UUID NOT NULL, + -- Exactly one home each. A manifest grows with the project and passes a few + -- hundred KB on a handful of models, so a large one goes to the INSTANCE's + -- object storage and this row keeps the key; a small one stays here, where it + -- costs no round trip and works on an instance that has configured no storage + -- at all. The instance's and not the workspace's, because a member can write + -- the workspace bucket under a key of their choosing, and a manifest is what a + -- later run resolves every unbuilt `ref()` through. `run_results.json` is a + -- tenth of the size and takes the same two homes rather than a rule of its own. + manifest TEXT, + manifest_key TEXT, + run_results TEXT, + run_results_key TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, environment), + CONSTRAINT dbt_environment_state_manifest_one_home + CHECK (num_nonnulls(manifest, manifest_key) = 1), + CONSTRAINT dbt_environment_state_run_results_one_home + CHECK (num_nonnulls(run_results, run_results_key) <= 1) +); + +-- No age sweep, unlike the per-run graph rows next door: this table holds one +-- row per script per environment and replaces it in place, so it does not grow +-- with runs, and its reader is every later run of that script — a project that +-- runs monthly must still find last month's state. It goes with the script +-- instead, alongside `dbt_run_state`. +GRANT ALL ON dbt_environment_state TO windmill_user; +GRANT ALL ON dbt_environment_state TO windmill_admin; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.down.sql b/backend/migrations/20260904143633_dbt_column_lineage.down.sql new file mode 100644 index 0000000000..a053a94291 --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dbt_column_edge; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS column_schema; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.up.sql b/backend/migrations/20260904143633_dbt_column_lineage.up.sql new file mode 100644 index 0000000000..cb463c249f --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.up.sql @@ -0,0 +1,68 @@ +-- Column-level lineage, from the engine's own static analysis. +-- +-- `manifest.json` carries none, which is why decision 14 recorded the feature as +-- unavailable. The edges exist in a different artifact: an engine that does +-- static analysis writes `target/index/dbt.column_lineage.parquet` under +-- `dbt compile --static-analysis strict --write-index`. That pass is opt-in per +-- project (`column_lineage: true`), because strict analysis rejects SQL the +-- default accepts and must never become a silent requirement of running a build. + +-- One column-to-column edge, keyed exactly like `dbt_edge`: a version's graph +-- dies with its version through the composite foreign key, a run's snapshot is +-- keyed by `job_id` with the zero UUID meaning "the version's own graph", and an +-- editor buffer's parse carries a NULL `script_hash` keyed to its preview job. +CREATE TABLE IF NOT EXISTS dbt_column_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + parent_column TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + child_column TEXT NOT NULL, + -- dbt's own word for how the value travelled: `copy` (passthrough), `mod` + -- (transformed), `scan` (the column was read to produce the ROW rather than + -- the value -- a join key, a `where` predicate, a `group by`). TEXT rather + -- than an enum because the engine treats the set as open: its own reader maps + -- those three and returns anything else verbatim. + lineage_kind TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Two partial unique indexes rather than a primary key, for the reason + -- 20260801121717 gives: a versioned graph is keyed by its version, a buffer + -- parse by its job alone. `lineage_kind` is part of the key because it is part + -- of the fact: a column that is both projected and used as a predicate for the + -- same output column has a `copy` edge AND a `scan` one, and the digest counts + -- both. Leaving it out let `ON CONFLICT DO NOTHING` drop the second while the + -- digest still claimed it was stored. + CONSTRAINT dbt_column_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_versioned_key + ON dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_editor_key + ON dbt_column_edge (workspace_id, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NULL; + +-- Same age sweep as the other per-run rows, and the same reason there is no +-- foreign key to `v2_job`. +CREATE INDEX IF NOT EXISTS idx_dbt_column_edge_run_age ON dbt_column_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; + +-- The real column schema of a node, which only static analysis knows: an +-- ordered `[{"name": …, "type": …}]`, from `dbt.node_columns.parquet`. +-- +-- Beside `columns` rather than folded into it. `columns` is the DECLARED +-- metadata `manifest.json` carries -- the names an author wrote in `schema.yml` +-- and the prose against them -- and stays exactly that, so a project that +-- documents two of forty columns keeps saying so. This is the other forty, +-- typed, in the order the model produces them. +ALTER TABLE dbt_node ADD COLUMN IF NOT EXISTS column_schema JSONB; + +GRANT ALL ON dbt_column_edge TO windmill_user; +GRANT ALL ON dbt_column_edge TO windmill_admin; diff --git a/backend/migrations/20260908113909_native_trigger_enabled.down.sql b/backend/migrations/20260908113909_native_trigger_enabled.down.sql new file mode 100644 index 0000000000..75f26b22c4 --- /dev/null +++ b/backend/migrations/20260908113909_native_trigger_enabled.down.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger DROP COLUMN IF EXISTS enabled; diff --git a/backend/migrations/20260908113909_native_trigger_enabled.up.sql b/backend/migrations/20260908113909_native_trigger_enabled.up.sql new file mode 100644 index 0000000000..15139498d7 --- /dev/null +++ b/backend/migrations/20260908113909_native_trigger_enabled.up.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger ADD COLUMN enabled BOOLEAN NOT NULL DEFAULT true; diff --git a/backend/migrations/20260908142148_datatable_role_catalog.down.sql b/backend/migrations/20260908142148_datatable_role_catalog.down.sql new file mode 100644 index 0000000000..8d3f12bc4a --- /dev/null +++ b/backend/migrations/20260908142148_datatable_role_catalog.down.sql @@ -0,0 +1,18 @@ +-- Refuse while the catalog holds anything. Each row is a live Postgres login with a password +-- only this table carries, so dropping it would leave credentials on the cluster that Windmill can +-- no longer disable, delete or even name — and re-applying could not recreate them, because the +-- role names would already be taken. Cleaning them up here is not an option either: dropping a +-- role means reassigning what it owns in *every* instance database, and a migration runs in one. +-- +-- Delete the roles through instance settings first; that path does the cluster work. +LOCK TABLE datatable_role IN ACCESS EXCLUSIVE MODE; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM datatable_role) THEN + RAISE EXCEPTION 'Cannot roll back: % data table role(s) still exist as Postgres logins. Delete them in instance settings first, which drops them from the cluster.', + (SELECT count(*) FROM datatable_role); + END IF; +END $$; + +DROP TABLE IF EXISTS datatable_role; diff --git a/backend/migrations/20260908142148_datatable_role_catalog.up.sql b/backend/migrations/20260908142148_datatable_role_catalog.up.sql new file mode 100644 index 0000000000..59cce4cc08 --- /dev/null +++ b/backend/migrations/20260908142148_datatable_role_catalog.up.sql @@ -0,0 +1,21 @@ +-- The instance's data table role catalog: one row per Postgres login Windmill created for data +-- table access. +-- +-- A table rather than a `global_settings` key, because the value is a set of live cluster +-- credentials and that table has generic read, list, write and CLI round-trip paths that know +-- nothing about what they are carrying. Every one of them is a way to leak the passwords or to +-- overwrite the catalog with a copy that has none, and a row nothing generic touches has none of +-- those. One row per role also makes two concurrent creates two inserts rather than a +-- read-modify-write over one document. +CREATE TABLE datatable_role ( + id VARCHAR(50) PRIMARY KEY, + -- The Postgres role name, verbatim. Unique because it is the cluster's own key. + name VARCHAR(63) NOT NULL UNIQUE, + enabled BOOLEAN NOT NULL DEFAULT true, + -- Generated by Windmill, never entered by anyone, and never leaves the server. + pwd TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +GRANT ALL ON datatable_role TO windmill_user; +GRANT ALL ON datatable_role TO windmill_admin; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.down.sql b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql new file mode 100644 index 0000000000..32ae30d2bd --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql @@ -0,0 +1,9 @@ +-- Dropping the column drops the primary key and the identity sequence with it, and +-- only marks the column dropped in the catalog rather than rewriting the table, so +-- this takes the ACCESS EXCLUSIVE lock but not the time. + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_column_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_graph_snapshot DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.up.sql b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql new file mode 100644 index 0000000000..399fe6cc5f --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql @@ -0,0 +1,31 @@ +-- These five had neither a PRIMARY KEY nor an explicit REPLICA IDENTITY, which makes +-- PostgreSQL reject UPDATE and DELETE on them under logical replication. +-- `deployment_metadata` and `metrics` are the other two, one migration each after this. +-- +-- The surrogate cannot be swapped for a natural key: every unique index on all five is +-- PARTIAL, split on `script_hash IS NULL`, and a partial index cannot back a primary +-- key. The partial uniques stay; they are what the ON CONFLICT clauses infer. +-- +-- Each ALTER rewrites its table under ACCESS EXCLUSIVE and holds it unavailable for +-- the rewrite. These five share a transaction because each is bounded by what a +-- workspace holds rather than by how long it has run, so none can grow into the one +-- that locks the rest; a transaction holds all its locks until it commits. +-- +-- An instance that cannot afford that lock at startup can set REPLICA IDENTITY FULL +-- on these tables instead, which unblocks replication by itself, and run these +-- idempotent ALTERs in a maintenance window first. + +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_node + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_column_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_graph_snapshot + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql new file mode 100644 index 0000000000..3ce4ae52a6 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql new file mode 100644 index 0000000000..951253ce64 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql @@ -0,0 +1,13 @@ +-- The sixth of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the one table in the set with no retention +-- sweep -- rows accumulate per deployed script hash, flow version and app version for +-- the life of the instance -- so on an instance that upgrades after years of deploys +-- its ACCESS EXCLUSIVE rewrite is the one that could hold the others locked. +-- +-- No natural key: each row is a script, flow OR app deployment, and the three unique +-- indexes are partial on exactly that split, so none of them covers every row. + +ALTER TABLE deployment_metadata + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.down.sql b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql new file mode 100644 index 0000000000..1ad428dc02 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE metrics DROP COLUMN IF EXISTS row_id; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.up.sql b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql new file mode 100644 index 0000000000..b863035990 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql @@ -0,0 +1,12 @@ +-- The last of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the largest (~400 MB / 750k rows on the +-- instance this was measured on, a steady state: `queue_%` rows, which are nearly all +-- of them, are swept at 14 days) and its ALTER rewrites it under ACCESS EXCLUSIVE. One +-- migration is one transaction, so alone it holds no lock on the others as it rewrites. +-- +-- The surrogate cannot be called `id` -- `metrics.id` holds the metric NAME. + +ALTER TABLE metrics + ADD COLUMN IF NOT EXISTS row_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql b/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql new file mode 100644 index 0000000000..a089085444 --- /dev/null +++ b/backend/migrations/20260909092950_add_git_sync_synced_head.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS git_sync_synced_head; diff --git a/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql b/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql new file mode 100644 index 0000000000..a0c81d2671 --- /dev/null +++ b/backend/migrations/20260909092950_add_git_sync_synced_head.up.sql @@ -0,0 +1,24 @@ +-- One row per commit a workspace has come to reflect on a branch, written when a +-- pull of that commit succeeds or a deploy push produces it. The "Windmill CI +-- tests" PR check reads it to know when a workspace reflects a PR head, and +-- records the head's CI test runs on it. Kept apart from `workspace_settings.git_sync.auto_pull.last_synced_sha`, +-- which decides whether the next poll pulls and is client-round-tripped settings. +CREATE TABLE git_sync_synced_head ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- Repository resource path without its `$res:` prefix. + repo_resource_path VARCHAR(255) NOT NULL, + branch VARCHAR(255) NOT NULL, + sha VARCHAR(64) NOT NULL, + -- 'pull' rows name the pull job; 'push' rows the deploy push job. + source VARCHAR(4) NOT NULL CHECK (source IN ('pull', 'push')), + job_id UUID, + synced_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- The head's own CI test suite, dispatched once the workspace reflects it and its + -- dependency jobs settled: the "Windmill CI tests" check reads exactly these runs. + tests_dispatched_at TIMESTAMPTZ, + ci_test_job_ids UUID[], + PRIMARY KEY (workspace_id, repo_resource_path, branch, sha) +); + +GRANT ALL ON git_sync_synced_head TO windmill_user; +GRANT ALL ON git_sync_synced_head TO windmill_admin; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql new file mode 100644 index 0000000000..3405d5a681 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.down.sql @@ -0,0 +1,7 @@ +DROP INDEX IF EXISTS index_app_version_on_app_id; + +DROP INDEX IF EXISTS index_app_script_on_app; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; diff --git a/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql new file mode 100644 index 0000000000..6426aec594 --- /dev/null +++ b/backend/migrations/20260909163047_workspace_delete_cascade_indexes.up.sql @@ -0,0 +1,32 @@ +-- The FK columns that cascade when a workspace's apps and flows are deleted. Unindexed, +-- Postgres seq-scans the whole child table once per deleted parent row, making a workspace +-- delete cost O(apps and flows deleted x rows in the instance). Fork deletion is where that +-- bites: a fork clones its parent's apps, flows and entire app version history. +-- +-- The workspace_runnable_dependencies pair are partial because the table's check constraint +-- makes app_path and flow_path mutually exclusive, halving each index -- the cascade's +-- equality on the path proves the predicate. The table's existing path indexes are partial on +-- script_hash, which the cascade does not constrain, so they cannot serve it. +-- +-- Dropped before built: a failed concurrent build leaves an invalid index that IF NOT EXISTS +-- would accept forever, unused by the planner yet still maintained on every write. +-- +-- No statement separators outside the statements below, comments included: the CONCURRENTLY +-- rewrite in windmill-api/src/db.rs splits the file on them and would run comment text as SQL. +DROP INDEX IF EXISTS index_app_version_on_app_id; + +CREATE INDEX index_app_version_on_app_id ON app_version (app_id); + +DROP INDEX IF EXISTS index_app_script_on_app; + +CREATE INDEX index_app_script_on_app ON app_script (app); + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_app_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_app_path + ON workspace_runnable_dependencies (app_path, workspace_id) WHERE app_path IS NOT NULL; + +DROP INDEX IF EXISTS index_workspace_runnable_dependencies_on_flow_path; + +CREATE INDEX index_workspace_runnable_dependencies_on_flow_path + ON workspace_runnable_dependencies (flow_path, workspace_id) WHERE flow_path IS NOT NULL; diff --git a/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql b/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql new file mode 100644 index 0000000000..3c67031c9f --- /dev/null +++ b/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cloud_onboarding_profile; diff --git a/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql b/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql new file mode 100644 index 0000000000..8e295ea6c6 --- /dev/null +++ b/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql @@ -0,0 +1,9 @@ +-- Context an invite carried about the account's owner, written at provisioning and read by +-- onboarding to tailor itself (skip the source question it knows the answer to, later +-- template picks and starter prompts). Free-form JSON so new fields need no migration. +CREATE TABLE cloud_onboarding_profile ( + email VARCHAR(255) PRIMARY KEY REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + profile JSONB NOT NULL, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql new file mode 100644 index 0000000000..f615c55463 --- /dev/null +++ b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.down.sql @@ -0,0 +1,6 @@ +-- Add down migration script here +-- Nothing to undo. The up migration gives a policy that only ever carried the address the +-- principal it runs as, and rewrites an address that disagreed with its principal. The previous +-- version reads both halves, so both results are correct for it too, and the addresses replaced +-- named an account other than the one the app runs as. +SELECT 1; diff --git a/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql new file mode 100644 index 0000000000..fe97d7529c --- /dev/null +++ b/backend/migrations/20260911085221_backfill_app_policy_on_behalf_of.up.sql @@ -0,0 +1,100 @@ +-- Add up migration script here +-- `policy.on_behalf_of` becomes the authority for an app's identity: the address beside it is +-- written through from it on every save, so the two can no longer name different accounts. +-- +-- The address key is deliberately NOT removed here, and is still written: a replica predating +-- the derive-when-absent fallback errors outright when it is missing, which would 400 every +-- anonymous, publisher and guest app served by one that has not yet rolled over. Removing the +-- key is a follow-up, per docs/app-policy-email-removal.md. +-- +-- What is left is the data written before that rule. A policy that only ever had the address has +-- no principal to run as, so give it one. A policy whose halves disagree was stored as a client +-- sent it; reads return that pair and a redeploy that keeps the identity sends it back, where the +-- pair check rejects it. So once every policy has a principal, rewrite its address from it. + +-- Mirrors `users::username_to_permissioned_as`: an email-shaped username is its own principal +-- unless it contains a slash, which a reader would split on, and a legacy `group-*` username is +-- the group it names. +CREATE OR REPLACE FUNCTION pg_temp.username_to_permissioned_as(name VARCHAR) +RETURNS VARCHAR AS $$ + SELECT CASE + WHEN $1 LIKE '%@%' AND $1 LIKE '%/%' THEN 'u/' || $1 + WHEN $1 LIKE '%@%' THEN $1 + WHEN $1 LIKE 'group-%' THEN 'g/' || substr($1, 7) + ELSE 'u/' || $1 + END; +$$ LANGUAGE SQL IMMUTABLE; + +-- Mirrors `users::permissioned_as_from_email`: a real account wins over the synthetic group +-- namespace, which is not reserved and may be a user's own address. `pg_temp` lives for the +-- whole session and migrations share one connection, so an identically-named helper from an +-- earlier migration is still in scope: replace it, and drop this one at the end. +CREATE OR REPLACE FUNCTION pg_temp.permissioned_as_from_email(w_id VARCHAR, email VARCHAR) +RETURNS VARCHAR AS $$ + SELECT COALESCE( + (SELECT pg_temp.username_to_permissioned_as(u.username) + FROM usr u WHERE u.workspace_id = $1 AND u.email = $2), + -- A superadmin acting outside their workspaces has no usr row. + (SELECT pg_temp.username_to_permissioned_as(COALESCE(p.username, p.email)) + FROM password p WHERE p.email = $2 AND p.super_admin), + (SELECT 'g/' || g.name FROM group_ g + WHERE g.workspace_id = $1 + AND $2 = 'group-' || g.name || '@windmill.dev') + ); +$$ LANGUAGE SQL STABLE; + +-- Mirrors `users::get_email_from_permissioned_as`, except that a `u/` principal naming nobody +-- yields NULL rather than the synthetic `@unknown.windmill.dev` address, so that row is left as +-- it is instead of losing the one address it had. +CREATE OR REPLACE FUNCTION pg_temp.email_from_permissioned_as(w_id VARCHAR, principal VARCHAR) +RETURNS VARCHAR AS $$ + SELECT CASE + WHEN $2 LIKE 'u/%' THEN COALESCE( + (SELECT u.email FROM usr u WHERE u.workspace_id = $1 AND u.username = substr($2, 3)), + (SELECT p.email FROM password p + WHERE (p.username = substr($2, 3) OR p.email = substr($2, 3)) AND p.super_admin + ORDER BY p.email LIMIT 1)) + WHEN $2 LIKE 'g/%' THEN 'group-' || substr($2, 3) || '@windmill.dev' + ELSE $2 + END; +$$ LANGUAGE SQL STABLE; + +-- A policy naming only the address predates the principal being written to it. +-- A principal wider than `v2_job.permissioned_as` could not be enqueued, so it is not recorded +-- at all — the app falls back to erroring on anonymous execution until someone picks an identity +-- the deploy path accepts. Same cap and reason as the sibling migration 20260801043001. +UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email'))) + WHERE policy->>'on_behalf_of' IS NULL + AND pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email') IS NOT NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, policy->>'on_behalf_of_email')) <= 55; + +-- App drafts carry a copy of the policy and are deployed from it, so they need the same. +UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], + to_jsonb(pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email')))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' IS NULL + AND pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email') IS NOT NULL + AND length(pg_temp.permissioned_as_from_email(workspace_id, value->'policy'->>'on_behalf_of_email')) <= 55; + +-- The address a save now writes, applied to the rows saved before. Execution already takes a `u/` +-- principal's own address, so this only changes what runs for a `g/` or bare principal, whose +-- stored address decided the superadmin flag and instance groups: those now follow the principal. +UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of_email'], + to_jsonb(pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of'))) + WHERE policy->>'on_behalf_of' IS NOT NULL + AND pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of') IS NOT NULL + AND policy->>'on_behalf_of_email' + IS DISTINCT FROM pg_temp.email_from_permissioned_as(workspace_id, policy->>'on_behalf_of'); + +UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], + to_jsonb(pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of')))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' IS NOT NULL + AND pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of') IS NOT NULL + AND value->'policy'->>'on_behalf_of_email' + IS DISTINCT FROM pg_temp.email_from_permissioned_as(workspace_id, value->'policy'->>'on_behalf_of'); + +DROP FUNCTION pg_temp.permissioned_as_from_email(VARCHAR, VARCHAR); +DROP FUNCTION pg_temp.email_from_permissioned_as(VARCHAR, VARCHAR); +DROP FUNCTION pg_temp.username_to_permissioned_as(VARCHAR); diff --git a/backend/migrations/20260911085230_notify_user_email_change.down.sql b/backend/migrations/20260911085230_notify_user_email_change.down.sql new file mode 100644 index 0000000000..7f6f494115 --- /dev/null +++ b/backend/migrations/20260911085230_notify_user_email_change.down.sql @@ -0,0 +1,8 @@ +-- Add down migration script here +DROP TRIGGER IF EXISTS password_superadmin_delete_trigger ON password; +DROP TRIGGER IF EXISTS password_superadmin_insert_trigger ON password; +DROP TRIGGER IF EXISTS password_superadmin_update_trigger ON password; +DROP TRIGGER IF EXISTS usr_email_update_trigger ON usr; +DROP TRIGGER IF EXISTS usr_email_change_trigger ON usr; +DROP FUNCTION IF EXISTS notify_superadmin_identity_change(); +DROP FUNCTION IF EXISTS notify_usr_email_change(); diff --git a/backend/migrations/20260911085230_notify_user_email_change.up.sql b/backend/migrations/20260911085230_notify_user_email_change.up.sql new file mode 100644 index 0000000000..f4e4a818f9 --- /dev/null +++ b/backend/migrations/20260911085230_notify_user_email_change.up.sql @@ -0,0 +1,85 @@ +-- Add up migration script here +-- Emit a notify_event so every process evicts its cached `permissioned_as` -> address mapping +-- (windmill-common EMAIL_CACHE) at its next notify-event poll, rather than serving the old +-- address for the rest of the TTL. Authorization does not rest on this: +-- `fetch_authed_from_permissioned_as` re-resolves the address from the principal's live binding. +-- SECURITY DEFINER so the INSERT runs as the function owner rather than the invoking +-- windmill_user/windmill_admin role, matching the other notify_* triggers. +CREATE OR REPLACE FUNCTION notify_usr_email_change() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO notify_event (channel, payload) + VALUES ( + 'notify_user_email_change', + COALESCE(NEW.workspace_id, OLD.workspace_id) || ':' || COALESCE(NEW.username, OLD.username) + ); + -- A rename leaves the OLD username cached against this account's address; evict both keys. + IF TG_OP = 'UPDATE' AND NEW.username IS DISTINCT FROM OLD.username THEN + INSERT INTO notify_event (channel, payload) + VALUES ('notify_user_email_change', OLD.workspace_id || ':' || OLD.username); + END IF; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +-- INSERT matters too: a lookup that resolved to nobody is cached as the synthetic +-- `{username}@unknown.windmill.dev`, so creating the row has to drop that entry. +CREATE TRIGGER usr_email_change_trigger +AFTER INSERT OR DELETE ON usr +FOR EACH ROW +EXECUTE FUNCTION notify_usr_email_change(); + +CREATE TRIGGER usr_email_update_trigger +AFTER UPDATE OF email, username ON usr +FOR EACH ROW +WHEN (OLD.email IS DISTINCT FROM NEW.email OR OLD.username IS DISTINCT FROM NEW.username) +EXECUTE FUNCTION notify_usr_email_change(); + +-- A superadmin acting outside their workspaces resolves through `password` instead, and that row +-- names no workspace of its own. The `*:` payload says so: the reader drops that name's entry in +-- every workspace rather than the whole cache, which would undo the caching on an instance that +-- rewrites these rows in bulk. Confined to superadmins because they are the only accounts the +-- `usr` triggers above cannot cover. +CREATE OR REPLACE FUNCTION notify_superadmin_identity_change() +RETURNS TRIGGER AS $$ +DECLARE + names TEXT[] := '{}'; +BEGIN + -- Every alias the principal can be spelled as: `resolve_username_to_email` matches a `u/` + -- principal against `username` OR `email`, and whichever string the caller passed is the key + -- it cached under, so one account can hold a live entry under either. Old and new of each, + -- because a change to one leaves the other's entry behind. + IF TG_OP <> 'DELETE' THEN names := names || ARRAY[NEW.username, NEW.email]; END IF; + IF TG_OP <> 'INSERT' THEN names := names || ARRAY[OLD.username, OLD.email]; END IF; + INSERT INTO notify_event (channel, payload) + SELECT DISTINCT 'notify_user_email_change', '*:' || n + FROM unnest(names) AS n + WHERE n IS NOT NULL; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +-- `super_admin` is half of what the fallback matches on, so gaining or losing it moves the +-- mapping as surely as the address does: a demotion leaves the real address cached where the +-- truth is now synthetic, and a promotion leaves that synthetic one cached in place of a real +-- account. `OLD.super_admin OR NEW.super_admin` is what catches both directions. +CREATE TRIGGER password_superadmin_update_trigger +AFTER UPDATE OF email, username, super_admin ON password +FOR EACH ROW +WHEN ((OLD.super_admin OR NEW.super_admin) + AND (OLD.email IS DISTINCT FROM NEW.email + OR OLD.username IS DISTINCT FROM NEW.username + OR OLD.super_admin IS DISTINCT FROM NEW.super_admin)) +EXECUTE FUNCTION notify_superadmin_identity_change(); + +CREATE TRIGGER password_superadmin_insert_trigger +AFTER INSERT ON password +FOR EACH ROW +WHEN (NEW.super_admin) +EXECUTE FUNCTION notify_superadmin_identity_change(); + +CREATE TRIGGER password_superadmin_delete_trigger +AFTER DELETE ON password +FOR EACH ROW +WHEN (OLD.super_admin) +EXECUTE FUNCTION notify_superadmin_identity_change(); diff --git a/backend/migrations/20260914131148_resource_type_display_name.down.sql b/backend/migrations/20260914131148_resource_type_display_name.down.sql new file mode 100644 index 0000000000..57c9ffabc8 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource_type DROP COLUMN display_name; diff --git a/backend/migrations/20260914131148_resource_type_display_name.up.sql b/backend/migrations/20260914131148_resource_type_display_name.up.sql new file mode 100644 index 0000000000..213622ef94 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.up.sql @@ -0,0 +1,24 @@ +-- The name a product goes by, beside the identifier a resource references: `gsheets` is +-- "Google Sheets". Null where nobody named the type; readers derive a label from the name. +ALTER TABLE resource_type ADD COLUMN display_name VARCHAR(100); + +-- The names the hub carries today, so existing instances show them before any sync. Only in +-- admins, where hub resource types live and every workspace reads them from. +UPDATE resource_type SET display_name = v.display_name +FROM (VALUES + ('bamboo_hr', 'BambooHR'), + ('cacertificate', 'CA certificate'), + ('deep_infra', 'DeepInfra'), + ('gcal', 'Google Calendar'), + ('gdocs', 'Google Docs'), + ('gdrive', 'Google Drive'), + ('gforms', 'Google Forms'), + ('gsheets', 'Google Sheets'), + ('gworkspace', 'Google Workspace'), + ('sensortower', 'Sensor Tower'), + ('snowflake_oauth', 'Snowflake (OAuth)'), + ('their_stack', 'TheirStack') +) AS v(name, display_name) +WHERE resource_type.workspace_id = 'admins' + AND resource_type.name = v.name + AND resource_type.display_name IS NULL; diff --git a/backend/migrations/20260914133558_draft_base_version.down.sql b/backend/migrations/20260914133558_draft_base_version.down.sql new file mode 100644 index 0000000000..07fc93ba30 --- /dev/null +++ b/backend/migrations/20260914133558_draft_base_version.down.sql @@ -0,0 +1 @@ +ALTER TABLE draft DROP COLUMN base; diff --git a/backend/migrations/20260914133558_draft_base_version.up.sql b/backend/migrations/20260914133558_draft_base_version.up.sql new file mode 100644 index 0000000000..835816d70c --- /dev/null +++ b/backend/migrations/20260914133558_draft_base_version.up.sql @@ -0,0 +1,14 @@ +-- The version a draft forked from, as one opaque text id whatever the kind: a +-- script hash (hex), a flow_version.id, an app_version.id. NULL for a draft that +-- was never forked from a deploy and for kinds that keep no lineage. +ALTER TABLE draft ADD COLUMN base TEXT; + +-- A U+0000 inside a `json` value makes `->>` raise 22P05; such rows keep NULL and +-- get their base on their next save. +UPDATE draft SET base = CASE typ::text + WHEN 'script' THEN value ->> 'parent_hash' + WHEN 'flow' THEN value ->> 'version_id' + ELSE value ->> 'parent_version' + END +WHERE typ::text IN ('script', 'flow', 'app', 'raw_app') + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0; diff --git a/backend/migrations/20260914135805_ai_shared_artifact.down.sql b/backend/migrations/20260914135805_ai_shared_artifact.down.sql new file mode 100644 index 0000000000..04fa92db6c --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS ai_shared_artifact; diff --git a/backend/migrations/20260914135805_ai_shared_artifact.up.sql b/backend/migrations/20260914135805_ai_shared_artifact.up.sql new file mode 100644 index 0000000000..6daa3e5daf --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.up.sql @@ -0,0 +1,32 @@ +-- A copy of an AI session artifact that its author explicitly shared with the workspace. +-- Artifacts otherwise live only in the author's browser; this row exists only while the +-- share does, and the monitor deletes it once `shared_at` falls outside +-- AI_SHARED_ARTIFACT_RETENTION_SECS. +CREATE TABLE ai_shared_artifact ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- The browser-side artifact id. Unique per author so sharing the same artifact again + -- moves its one link forward rather than minting a second one. + artifact_id VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL, + created_by VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + kind VARCHAR(10) NOT NULL CHECK (kind IN ('md', 'html')), + version INTEGER NOT NULL, + content TEXT NOT NULL, + -- Reset on every re-share: retention counts from the last time the author shared it. + shared_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (workspace_id, email, artifact_id) +); + +CREATE INDEX idx_ai_shared_artifact_shared_at ON ai_shared_artifact (shared_at); + +GRANT ALL ON ai_shared_artifact TO windmill_admin; +GRANT ALL ON ai_shared_artifact TO windmill_user; + +-- The handlers go through the raw pool and scope every query to the workspace themselves. +-- An admin-only policy is the backstop for a future query that reaches this table through +-- UserDB. +ALTER TABLE ai_shared_artifact ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_shared_artifact FOR ALL TO windmill_admin USING (true); diff --git a/backend/migrations/20260914154128_draft_move.down.sql b/backend/migrations/20260914154128_draft_move.down.sql new file mode 100644 index 0000000000..6fff84e4bb --- /dev/null +++ b/backend/migrations/20260914154128_draft_move.down.sql @@ -0,0 +1 @@ +DROP TABLE draft_move; diff --git a/backend/migrations/20260914154128_draft_move.up.sql b/backend/migrations/20260914154128_draft_move.up.sql new file mode 100644 index 0000000000..e68e36c5ff --- /dev/null +++ b/backend/migrations/20260914154128_draft_move.up.sql @@ -0,0 +1,17 @@ +-- Where an item's drafts went when it moved, so a draft save still addressed to the +-- old path (an editor left open across the move, a chat, the CLI) lands on the moved +-- draft instead of starting a new item there. `email` NULL records a deployed item's +-- move and applies to every user; set, it records that user's draft-only move. +CREATE TABLE draft_move ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + typ DRAFT_KIND NOT NULL, + old_path VARCHAR(255) NOT NULL, + new_path VARCHAR(255) NOT NULL, + email VARCHAR(255) +); + +CREATE INDEX draft_move_old_path_idx ON draft_move (workspace_id, typ, old_path); + +GRANT ALL ON draft_move TO windmill_user; +GRANT ALL ON draft_move TO windmill_admin; diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql new file mode 100644 index 0000000000..2662a9f709 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS ai_sessions_backup_generation; diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql new file mode 100644 index 0000000000..a62caefa60 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql @@ -0,0 +1,4 @@ +-- Bumped by every workspace key rotation: the AI session backups in the workspace storage +-- live under a prefix named by it, so a rotation moves to a fresh prefix and the previous +-- ones can be deleted at leisure without ever touching live objects. +ALTER TABLE workspace_settings ADD COLUMN ai_sessions_backup_generation BIGINT NOT NULL DEFAULT 0; diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql new file mode 100644 index 0000000000..c749fbe1b7 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql @@ -0,0 +1,9 @@ +DELETE FROM workspace_runnable_dependencies WHERE runnable_is_agent; + +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, workspace_id) + WHERE script_hash IS NULL; + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN runnable_is_agent; diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql new file mode 100644 index 0000000000..eab08e5c17 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql @@ -0,0 +1,21 @@ +-- A flow step linked to a saved agent (an `ai_agent` resource) is recorded next to the scripts and +-- subflows the flow runs, so renaming the agent can name the flows it would break. An agent row is +-- neither a script nor a flow: readers of script usages have to exclude it. +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN runnable_is_agent BOOLEAN NOT NULL DEFAULT false; + +-- A script step and a linked agent can share a path. Without the flag in the key, the second +-- insert's ON CONFLICT DO NOTHING would silently drop one of the two rows. +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) + WHERE script_hash IS NULL; + +-- The worker only records a flow when it is next deployed, so seed the ones already linking an +-- agent from their current value. +INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) +SELECT DISTINCT f.path, agent_ref #>> '{}', false, true, f.workspace_id +FROM flow f +CROSS JOIN LATERAL jsonb_path_query(f.value, 'lax $.** ? (@.type == "aiagent" && @.agent.type() == "string").agent') AS agent_ref +ON CONFLICT DO NOTHING; diff --git a/backend/migrations/20260916144653_flow_conversation_message_tool_call.down.sql b/backend/migrations/20260916144653_flow_conversation_message_tool_call.down.sql new file mode 100644 index 0000000000..56f20ba722 --- /dev/null +++ b/backend/migrations/20260916144653_flow_conversation_message_tool_call.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE flow_conversation_message DROP COLUMN tool_arguments; +ALTER TABLE flow_conversation_message DROP COLUMN tool_result; +ALTER TABLE flow_conversation_message DROP COLUMN reasoning; diff --git a/backend/migrations/20260916144653_flow_conversation_message_tool_call.up.sql b/backend/migrations/20260916144653_flow_conversation_message_tool_call.up.sql new file mode 100644 index 0000000000..a6f7822246 --- /dev/null +++ b/backend/migrations/20260916144653_flow_conversation_message_tool_call.up.sql @@ -0,0 +1,12 @@ +-- A chat is rebuilt from its rows without reading jobs, so every tool row carries its call: +-- the arguments the model wrote and the text the model got back, or what the call failed +-- with. A script or flow tool's job holds the args its input transforms produced, not the +-- model's; an MCP tool runs inside the agent's job, whose result lists every call of the +-- turn with nothing tying one to a row. A provider-native web search carries only its +-- citations, the provider never returning the query. +ALTER TABLE flow_conversation_message ADD COLUMN tool_arguments TEXT; +ALTER TABLE flow_conversation_message ADD COLUMN tool_result TEXT; + +-- The thinking behind this row. The agent job keeps the turn's thinking as one string; +-- the rows keep it per iteration, next to the answer or tool call it led to. +ALTER TABLE flow_conversation_message ADD COLUMN reasoning TEXT; diff --git a/backend/migrations/20260916155738_flow_conversation_is_test.down.sql b/backend/migrations/20260916155738_flow_conversation_is_test.down.sql new file mode 100644 index 0000000000..aa186105cf --- /dev/null +++ b/backend/migrations/20260916155738_flow_conversation_is_test.down.sql @@ -0,0 +1 @@ +ALTER TABLE flow_conversation DROP COLUMN is_test; diff --git a/backend/migrations/20260916155738_flow_conversation_is_test.up.sql b/backend/migrations/20260916155738_flow_conversation_is_test.up.sql new file mode 100644 index 0000000000..970375a84d --- /dev/null +++ b/backend/migrations/20260916155738_flow_conversation_is_test.up.sql @@ -0,0 +1,26 @@ +-- A chat run from the flow editor's test panel is stored exactly like one from the +-- deployed flow, so the two were indistinguishable once written. Marking them lets the +-- lists tell a trial apart from a real conversation. +ALTER TABLE flow_conversation ADD COLUMN is_test BOOLEAN NOT NULL DEFAULT false; + +-- Existing rows: a conversation whose messages came from a flowpreview run was a test. +-- Derived once here because the job is purged on retention, after which the origin of an +-- old conversation is unknowable. +-- +-- Walked to the root job rather than matched directly: an existing message row never holds +-- the flow job itself. The rows point at the step that produced them — the AI agent's job +-- for an answer, the tool's own job for a tool call — whose kind is never 'flowpreview'. +-- +-- `root_job` first, matching `get_root_job_id` (windmill-worker/src/common.rs): only it +-- reaches the top of the run. `flow_innermost_root_job` stops at the closest flow scope by +-- design, so an agent inside a subflow would land on that subflow's 'flow' row and the +-- conversation would read as deployed. +UPDATE flow_conversation c +SET is_test = true +WHERE EXISTS ( + SELECT 1 FROM flow_conversation_message m + JOIN v2_job j ON j.id = m.job_id + JOIN v2_job root + ON root.id = coalesce(j.root_job, j.flow_innermost_root_job, j.parent_job, j.id) + WHERE m.conversation_id = c.id AND root.kind = 'flowpreview' +); diff --git a/backend/migrations/20260917074040_flow_conversation_message_attachments.down.sql b/backend/migrations/20260917074040_flow_conversation_message_attachments.down.sql new file mode 100644 index 0000000000..01d81a9719 --- /dev/null +++ b/backend/migrations/20260917074040_flow_conversation_message_attachments.down.sql @@ -0,0 +1 @@ +ALTER TABLE flow_conversation_message DROP COLUMN attachments; diff --git a/backend/migrations/20260917074040_flow_conversation_message_attachments.up.sql b/backend/migrations/20260917074040_flow_conversation_message_attachments.up.sql new file mode 100644 index 0000000000..dec5365f89 --- /dev/null +++ b/backend/migrations/20260917074040_flow_conversation_message_attachments.up.sql @@ -0,0 +1,4 @@ +-- The files a user message carried, as object-storage references: `[{input, s3, storage?, +-- filename?}]`. Only references, never file bytes and never a presigned URL, so a +-- transcript can show a message's files without reading its run's args. +ALTER TABLE flow_conversation_message ADD COLUMN attachments JSONB; diff --git a/backend/migrations/20260918092041_add_admins_and_developers_to_forks.down.sql b/backend/migrations/20260918092041_add_admins_and_developers_to_forks.down.sql new file mode 100644 index 0000000000..4e8ac47c69 --- /dev/null +++ b/backend/migrations/20260918092041_add_admins_and_developers_to_forks.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN add_admins_and_developers_to_forks; diff --git a/backend/migrations/20260918092041_add_admins_and_developers_to_forks.up.sql b/backend/migrations/20260918092041_add_admins_and_developers_to_forks.up.sql new file mode 100644 index 0000000000..bb3502d362 --- /dev/null +++ b/backend/migrations/20260918092041_add_admins_and_developers_to_forks.up.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings ADD COLUMN add_admins_and_developers_to_forks BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index cb874eb2c7..bc2498a943 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -29,6 +29,10 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/spreadsheets"], + "scope_options": [ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/spreadsheets.readonly" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -38,6 +42,11 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/drive"], + "scope_options": [ + "https://www.googleapis.com/auth/drive.file", + "https://www.googleapis.com/auth/drive.readonly", + "https://www.googleapis.com/auth/drive" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -47,6 +56,13 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/gmail.send"], + "scope_options": [ + "https://www.googleapis.com/auth/gmail.send", + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/gmail.compose", + "https://www.googleapis.com/auth/gmail.modify", + "https://www.googleapis.com/auth/gmail.labels" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -56,6 +72,12 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/calendar.events"], + "scope_options": [ + "https://www.googleapis.com/auth/calendar.events", + "https://www.googleapis.com/auth/calendar.events.readonly", + "https://www.googleapis.com/auth/calendar.readonly", + "https://www.googleapis.com/auth/calendar" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -65,6 +87,12 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/forms"], + "scope_options": [ + "https://www.googleapis.com/auth/forms", + "https://www.googleapis.com/auth/forms.body", + "https://www.googleapis.com/auth/forms.body.readonly", + "https://www.googleapis.com/auth/forms.responses.readonly" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -74,6 +102,10 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "scopes": ["https://www.googleapis.com/auth/cloud-platform"], + "scope_options": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -88,6 +120,15 @@ "https://www.googleapis.com/auth/admin.directory.user.security", "https://www.googleapis.com/auth/admin.directory.orgunit" ], + "scope_options": [ + "https://www.googleapis.com/auth/admin.directory.user", + "https://www.googleapis.com/auth/admin.directory.user.readonly", + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.readonly", + "https://www.googleapis.com/auth/admin.directory.orgunit", + "https://www.googleapis.com/auth/admin.directory.orgunit.readonly", + "https://www.googleapis.com/auth/admin.directory.user.security" + ], "extra_params": { "access_type": "offline", "prompt": "consent" @@ -200,6 +241,7 @@ } }, "snowflake_oauth": { + "resource_fields": ["database", "warehouse", "role", "schema"], "connect_config_template": { "display_name": "Snowflake", "label": "Snowflake Account Identifier", diff --git a/backend/oauth_login.json b/backend/oauth_login.json index 93cdaac2b2..4706429404 100644 --- a/backend/oauth_login.json +++ b/backend/oauth_login.json @@ -15,13 +15,19 @@ "auth_url": "https://accounts.google.com/o/oauth2/v2/auth", "token_url": "https://oauth2.googleapis.com/token", "userinfo_url": "https://www.googleapis.com/oauth2/v1/userinfo?alt=json", - "scopes": ["https://www.googleapis.com/auth/userinfo.email"] + "scopes": ["https://www.googleapis.com/auth/userinfo.email"], + "extra_params": { + "prompt": "select_account" + } }, "microsoft": { "auth_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", "token_url": "https://login.microsoftonline.com/common/oauth2/v2.0/token", "userinfo_url": "https://graph.microsoft.com/oidc/userinfo", - "scopes": ["openid", "profile", "email"] + "scopes": ["openid", "profile", "email"], + "extra_params": { + "prompt": "select_account" + } }, "jumpcloud": { "auth_url": "https://oauth.id.jumpcloud.com/oauth2/auth", diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index b0ed12fb1a..271ad4835b 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -694,6 +694,107 @@ pub fn remove_pinned_imports(code: &str) -> anyhow::Result { Ok(content) } +/// Spans of the string literals naming a loaded module: `import`/`export … from` sources and the +/// argument of a dynamic `import()`. A `require()` call is left out: `require` is an ordinary +/// binding a script can shadow, so its argument is not known to be a module. +struct ImportSpecifierSpans(Vec); + +impl Visit for ImportSpecifierSpans { + noop_visit_type!(); + + fn visit_import_decl(&mut self, n: &swc_ecma_ast::ImportDecl) { + self.0.push(n.src.span); + } + + fn visit_export_all(&mut self, n: &swc_ecma_ast::ExportAll) { + self.0.push(n.src.span); + } + + fn visit_named_export(&mut self, n: &swc_ecma_ast::NamedExport) { + if let Some(src) = &n.src { + self.0.push(src.span); + } + } + + fn visit_call_expr(&mut self, n: &swc_ecma_ast::CallExpr) { + if let (swc_ecma_ast::Callee::Import(_), Some(arg)) = (&n.callee, n.args.first()) { + if let (None, Expr::Lit(Lit::Str(s))) = (arg.spread, &*arg.expr) { + self.0.push(s.span); + } + } + n.visit_children_with(self); + } +} + +/// Drops the `@version` from each pinned module specifier (`pkg@1.2.3/sub` -> `pkg/sub`), +/// rewriting only the specifier literals. Unlike [`remove_pinned_imports`], the same text +/// elsewhere, such as a string the script returns, stays as written. +pub fn remove_pinned_import_specifiers(code: &str) -> anyhow::Result { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file( + FileName::Custom("main.d.ts".into()).into(), + code.to_string(), + ); + let mut tss = TsSyntax::default(); + tss.tsx = true; + tss.no_early_errors = true; + let lexer = Lexer::new( + Syntax::Typescript(tss), + Default::default(), + StringInput::from(&*fm), + None, + ); + let module = Parser::new_from(lexer).parse_module().map_err(|e| { + anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {e:?}") + })?; + let mut specifiers = ImportSpecifierSpans(vec![]); + specifiers.visit_module(&module); + specifiers.0.sort_by_key(|s| s.lo); + + // Spans index the parsed source, which the source map stripped of any UTF-8 BOM. + let bom = if code.starts_with('\u{feff}') { + '\u{feff}'.len_utf8() + } else { + 0 + }; + let offset = + |pos: swc_common::BytePos| pos.0.checked_sub(fm.start_pos.0).map(|o| bom + o as usize); + let mut content = String::with_capacity(code.len()); + let mut copied = 0; + for span in specifiers.0 { + // A span covers the literal's quotes. One that does not land on a matching pair is left + // as written rather than risk rewriting the wrong bytes. + let (Some(open), Some(close)) = ( + offset(span.lo), + offset(span.hi).and_then(|e| e.checked_sub(1)), + ) else { + continue; + }; + let quote = code.as_bytes().get(open); + if open >= close + || open < copied + || !matches!(quote, Some(b'"' | b'\'')) + || code.as_bytes().get(close) != quote + { + continue; + } + let Some(specifier) = code.get(open + 1..close) else { + continue; + }; + let unpinned = IMPORTS_VERSION.captures(specifier).and_then(|x| { + x.get(1) + .map(|y| format!("{}{}", y.as_str(), x.get(2).map_or("", |z| z.as_str()))) + }); + if let Some(unpinned) = unpinned.filter(|u| u != specifier) { + content.push_str(&code[copied..open + 1]); + content.push_str(&unpinned); + copied = close; + } + } + content.push_str(&code[copied..]); + Ok(content) +} + fn resolve_type_ref(type_resolver: &HashMap, typ: &mut Typ) { let mut visited = std::collections::HashSet::new(); resolve_type_ref_with_visited(type_resolver, typ, &mut visited); diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 4309018fb4..4206dd68f8 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -4,6 +4,7 @@ mod tests { use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; use windmill_parser_ts::{ parse_deno_signature, parse_expr_for_imports, parse_relative_imports, + remove_pinned_import_specifiers, }; #[test] @@ -33,6 +34,46 @@ mod tests { ); } + #[test] + fn test_remove_pinned_import_specifiers_rewrites_only_specifiers() { + let code = r#"// héllo +import a from "pkg@1.2.3"; +import b from "@scope/pkg@^2/sub"; +export * from "other@3"; +import rel from "./helper"; +const c = await import("dyn@4"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"#; + assert_eq!( + remove_pinned_import_specifiers(code).unwrap(), + r#"// héllo +import a from "pkg"; +import b from "@scope/pkg/sub"; +export * from "other"; +import rel from "./helper"; +const c = await import("dyn"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"# + ); + assert_eq!( + remove_pinned_import_specifiers("\u{feff}import a from 'pkg@1';").unwrap(), + "\u{feff}import a from 'pkg';" + ); + assert_eq!( + remove_pinned_import_specifiers( + "// a\r\n// b\r\nimport a from \"pkg@1\";\r\nimport b from 'x@2';" + ) + .unwrap(), + "// a\r\n// b\r\nimport a from \"pkg\";\r\nimport b from 'x';" + ); + } + #[test] fn test_parse_empty_main_signature() { let code = r#" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index b6d83a9efb..83f0414e86 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -267,6 +267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -374,12 +375,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.7" @@ -1337,7 +1332,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin", ] [[package]] @@ -2186,16 +2181,20 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64 0.21.7", + "aws-lc-rs", + "base64 0.22.1", + "getrandom 0.2.17", + "js-sys", "pem", - "ring 0.16.20", "serde", "serde_json", + "signature", "simple_asn1", + "zeroize", ] [[package]] @@ -2225,7 +2224,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin", ] [[package]] @@ -2598,7 +2597,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin 0.9.8", + "spin", "version_check", ] @@ -2994,11 +2993,12 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pem" -version = "1.1.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "serde_core", ] [[package]] @@ -3403,7 +3403,7 @@ dependencies = [ "getrandom 0.3.4", "lru-slab", "rand 0.9.0", - "ring 0.17.14", + "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", @@ -3755,21 +3755,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.14" @@ -3901,7 +3886,7 @@ checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "once_cell", - "ring 0.17.14", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3964,7 +3949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "aws-lc-rs", - "ring 0.17.14", + "ring", "rustls-pki-types", "untrusted 0.9.0", ] @@ -4199,15 +4184,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -4426,12 +4412,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -6191,7 +6171,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.798.1" +version = "1.816.0" dependencies = [ "aho-corasick", "anyhow", @@ -6233,6 +6213,7 @@ dependencies = [ "pep440_rs", "phf 0.11.3", "pin-project-lite", + "pkcs1", "postgres-native-tls", "quick_cache", "rand 0.9.0", @@ -6244,8 +6225,10 @@ dependencies = [ "serde", "serde_json", "serde_yml", + "sha1", "sha2 0.10.9", "size", + "spki", "sqlx", "strum 0.27.2", "strum_macros 0.27.2", @@ -6274,7 +6257,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.798.1" +version = "1.816.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6269,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.798.1" +version = "1.816.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6278,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6290,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6326,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6338,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6349,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6360,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6372,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6431,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6461,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6473,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6491,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6507,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6523,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6555,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6569,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.798.1" +version = "1.816.0" dependencies = [ "anyhow", "bitflags", @@ -7282,6 +7265,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" @@ -7315,3 +7312,9 @@ dependencies = [ "quote", "syn 2.0.117", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 7296524140..1eacbd24d3 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.798.1" +version = "1.816.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs index 24f1e2c39d..36617c5639 100644 --- a/backend/parsers/windmill-parser-yaml/src/dbt.rs +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -58,6 +58,31 @@ impl DbtEngine { pub fn emits_node_events(&self) -> bool { matches!(self, DbtEngine::DbtCore1x) } + + /// Whether the engine's CLI has `--write-index`, the flag that writes the + /// parquet index column lineage lives in. False for 1.x, whose Python CLI + /// has no such option. + /// + /// True is not a promise that the artifact appears: `dbt-core` 2.0.0-alpha.5 + /// accepts the flag, declares the views over `dbt.column_lineage` in its own + /// `views.sql`, and writes neither that parquet nor `dbt.node_columns`. Only + /// Fusion does today. Attempting the pass on both is what lets a later 2.x + /// release pick the feature up with no change here. + pub fn writes_column_index(&self) -> bool { + !matches!(self, DbtEngine::DbtCore1x) + } + + /// Whether the engine has `--defer-state`, the deferral-only half of + /// `--state`. + /// + /// It matters on one command. `dbt retry` reads the run it resumes from + /// `--state`, so an engine with only that flag cannot be told to defer and + /// to resume from the job's own results at once: handed the deferral's + /// directory, it resumes the all-green run stored there and rebuilds + /// nothing. Only dbt-core 1.x separates the two. + pub fn has_defer_state_flag(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } } /// How the warehouse connection is supplied. Both paths are supported @@ -127,6 +152,18 @@ pub struct DbtDescriptor { pub selector: Option, #[serde(default)] pub test_behavior: DbtTestBehavior, + /// Ingest column-to-column lineage and the real column schemas, from the + /// engine's static analysis. + /// + /// Opt-in, and it has to be: the artifact only appears under + /// `--static-analysis strict`, which rejects SQL the default accepts (an + /// unresolvable identifier is an error there and compiles fine otherwise). + /// Turning it on for everyone would make a stricter dialect the price of + /// deploying a dbt project. It is a separate `dbt compile` pass, so nothing + /// it decides can change what a build does; a project it cannot analyze + /// keeps the graph it has today. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub column_lineage: bool, /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are /// all normal — so values keep their YAML type; only string leaves carry /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing @@ -137,6 +174,16 @@ pub struct DbtDescriptor { pub threads: Option, #[serde(default)] pub full_refresh: bool, + /// Resolve a `ref()` a run does not build through the state the last + /// successful run of this environment published, rather than through the + /// schema that run writes into. + /// + /// Only the default for the `build` block's own `defer`, since the choice is + /// per run: the run that publishes an environment's state and the run that + /// defers to it are two invocations of ONE script (decision 6), so a project + /// that could only defer by descriptor could never populate what it reads. + #[serde(default)] + pub defer: bool, /// Automatic in-job retry of the nodes a build failed on. /// /// dbt already confines a failure to its own subtree, and `dbt retry` @@ -258,6 +305,7 @@ pub const RESERVED_ARG_NAMES: &[&str] = &[ "exclude", "vars", "full_refresh", + "defer", "dbt_command", "dbt_retry_job", "model", @@ -345,15 +393,26 @@ fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { "build", selection() .into_iter() - .chain([Arg { - name: "full_refresh".to_string(), - otyp: None, - typ: Typ::Bool, - has_default: true, - default: Some(serde_json::json!(d.full_refresh)), - oidx: None, - otyp_inferred: false, - }]) + .chain([ + Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }, + Arg { + name: "defer".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.defer)), + oidx: None, + otyp_inferred: false, + }, + ]) .collect(), ), ( @@ -546,7 +605,9 @@ fn property_of(arg: &Arg) -> serde_json::Value { ), "select" => Some( "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ - `config.materialized:incremental`. Empty runs the descriptor's own selection.", + `config.materialized:incremental`. `state:modified+` and `result:error+` \ + compare against the state a previous run published, so they need `defer` on. \ + Empty runs the descriptor's own selection.", ), "exclude" => Some("Nodes to leave out of the selection above, same syntax."), "vars" => Some( @@ -554,6 +615,11 @@ fn property_of(arg: &Arg) -> serde_json::Value { exist makes this run store its own graph rather than the deployed one.", ), "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "defer" => Some( + "Resolve a `ref()` this run does not build to the relation the last successful \ + run of this warehouse and target published, instead of to the schema this run \ + writes into.", + ), "model" => Some( "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ two packages share a name. Any dbt selector resolving to ONE node works.", @@ -687,8 +753,15 @@ full_refresh: true }; let (build, build_args) = of("build"); - assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!( + build_args, + ["defer", "exclude", "full_refresh", "select", "vars"] + ); assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // `defer` is a per-run toggle rather than a descriptor-only setting: the + // run that publishes an environment's state and the run that defers to + // it are two invocations of ONE script. + assert_eq!(build["properties"]["defer"]["type"], "boolean"); // Defaults come from the descriptor, so an untouched run reproduces it. assert_eq!( build["properties"]["select"]["default"], diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..70fe5aa28a 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -728,7 +730,12 @@ pub fn parse_asset_syntax( s: &str, enable_default_syntax: bool, ) -> Option<(AssetKind, Cow<'_, str>)> { - if enable_default_syntax && s == "datatable" { + // `datatable` and `datatable?role=analyst` both name the default data table: the role picks + // which Postgres login the connection is made as, not which data table is read. + if enable_default_syntax + && s.strip_prefix("datatable") + .is_some_and(|rest| rest.is_empty() || rest.starts_with('?')) + { return Some((AssetKind::DataTable, Cow::Borrowed("main"))); } else if enable_default_syntax && s == "ducklake" { return Some((AssetKind::Ducklake, Cow::Borrowed("main"))); @@ -739,6 +746,14 @@ pub fn parse_asset_syntax( if *kind == AssetKind::Dbt { return Some((*kind, Cow::Owned(canonicalize_table_asset_path(suffix)))); } + // Same reasoning as above, for the explicit form. Specific to data tables: a + // `Resource`'s `?table=` is part of what it names, and stripping it would merge two + // different assets. + if *kind == AssetKind::DataTable { + if let Some((path, _role)) = suffix.split_once('?') { + return Some((*kind, Cow::Borrowed(path))); + } + } // The suffix is kept verbatim. For S3 the path encodes the storage: // `s3:///`, with an EMPTY storage segment for the // workspace default — so `s3:///key` yields `/key` (leading slash @@ -804,6 +819,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1678,6 +1705,25 @@ fn parse_trigger_spec(s: &str) -> Option { mod pipeline_annotation_tests { use super::*; + #[test] + fn a_datatable_role_is_not_part_of_the_asset_it_names() { + // The role picks which Postgres login the connection is made as, so two references that + // differ only by role are the same asset and must land on one graph node. + assert_eq!( + parse_asset_syntax("datatable://sales?role=analytics", false), + Some((AssetKind::DataTable, Cow::Borrowed("sales"))) + ); + assert_eq!( + parse_asset_syntax("datatable?role=analytics", true), + Some((AssetKind::DataTable, Cow::Borrowed("main"))) + ); + // A resource's `?table=` is part of what it names, so it is kept. + assert_eq!( + parse_asset_syntax("$res:f/db/pg?table=users", false), + Some((AssetKind::Resource, Cow::Borrowed("f/db/pg?table=users"))) + ); + } + #[test] fn s3_path_keeps_storage_distinction() { // An S3 asset path is `/` with an empty storage segment @@ -1740,10 +1786,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1808,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1845,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1864,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1884,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/src/main.rs b/backend/src/main.rs index f8b150383a..9e7a93cfcd 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,14 +7,14 @@ */ use anyhow::Context; use monitor::{ - load_base_url, load_otel, reload_critical_alerts_on_db_oversize, - reload_delete_logs_periodically_setting, reload_indexer_config, - reload_instance_python_version_setting, reload_maven_repos_setting, + flush_pending_log_files_to_object_store, load_base_url, load_otel, + reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, + reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, reload_maven_settings_xml_setting, reload_no_default_maven_setting, reload_nuget_config_setting, reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, reload_workspace_registries_setting, - send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, + send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::{Pool, Postgres}; @@ -46,23 +46,26 @@ use windmill_common::{ CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, + MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, + NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, @@ -124,7 +127,8 @@ use windmill_worker::{ use crate::monitor::{ initial_load, load_concurrency_key_max_queued, load_disable_password_login, - load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, load_metrics_debug_enabled, + load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, + load_mcp_disable_token_query_param, load_metrics_debug_enabled, load_preview_tags_override, load_require_preexisting_user, load_retention_period_overrides, load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, load_workspace_fairness_duration_secs, load_workspace_fairness_enabled, @@ -134,16 +138,18 @@ use crate::monitor::{ reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, - reload_extra_pip_index_url_setting, reload_http_route_workspaced_route_setting, + reload_extra_pip_index_url_setting, reload_http_route_default_allowed_origins_setting, + reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, - reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, - reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, - reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, + reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_sandbox_image_cache_max_setting, + reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, + reload_sandbox_image_pull_policy_setting, reload_sandbox_registry_auth_setting, + reload_scim_token_setting, reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -380,6 +386,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "", &mut None, &None, + None, ) .await { @@ -403,8 +410,19 @@ struct HubResourceTypeRaw { pub schema: Option, pub app: String, pub description: Option, + /// Absent from hubs predating the column, and from caches written before it. + #[serde(default)] + pub format_extension: Option, + /// Doubly optional, so a hub predating the field (no key) is told apart from a type the + /// hub leaves unnamed (null). + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } + /// Processed resource type with parsed schema #[derive(serde::Deserialize, serde::Serialize, Clone)] pub struct HubResourceType { @@ -413,6 +431,26 @@ pub struct HubResourceType { pub schema: Option, pub app: String, pub description: Option, + /// Doubly optional on purpose. A cache written before this column has no key at + /// all (`None`) and must leave the stored extension alone; one written since + /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely + /// dropping it and must clear. A single `Option` conflates the two, and picking + /// either meaning breaks the other — as does plain serde, which folds `null` + /// into the outer `None`, hence the wrapping deserializer. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, + /// Doubly optional like `format_extension`: a cache written before the field leaves the + /// stored name alone, while a null from the hub clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -459,6 +497,8 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { schema, app: rt.app, description: rt.description, + format_extension: Some(rt.format_extension), + display_name: rt.display_name, }) }) .collect(); @@ -500,9 +540,18 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh tracing::info!("Found {} cached resource types", cached_types.len()); - // Get existing resource types in admins workspace - let existing_types: Vec<(String, Option, Option)> = sqlx::query_as( - "SELECT name, schema, description FROM resource_type WHERE workspace_id = 'admins'", + // Get existing resource types in admins workspace. `format_extension` is part of + // the comparison below, so a type whose only change is gaining or losing it is + // not mistaken for unchanged; `is_fileset` decides whether it may take one. + let existing_types: Vec<( + String, + Option, + Option, + Option, + bool, + Option, + )> = sqlx::query_as( + "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -510,19 +559,68 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option), + ( + Option, + Option, + Option, + bool, + Option, + ), > = existing_types .into_iter() - .map(|(name, schema, desc)| (name, (schema, desc))) + .map( + |(name, schema, desc, format_extension, is_fileset, display_name)| { + ( + name, + (schema, desc, format_extension, is_fileset, display_name), + ) + }, + ) .collect(); let mut synced_count = 0; let mut skipped_count = 0; for rt in cached_types { - // Check if resource type already exists with same schema and description - if let Some((existing_schema, existing_desc)) = existing_map.get(&rt.name) { - if existing_schema == &rt.schema && existing_desc == &rt.description { + let existing = existing_map.get(&rt.name); + let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); + let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); + // A fileset is a set of files, so it cannot also be one file. Create, update + // and the manual sync all reject the pair; this writer would otherwise + // persist it onto a same-named local fileset. + // + // A cache with no key at all leaves the stored value alone, so the target is + // what is already there — which is also what makes the comparison below + // agree with the write instead of re-upserting the row on every boot. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(from_cache) => from_cache.clone(), + None => stored_extension.clone(), + } + }; + // No key in the cache leaves the stored name alone, as for the extension. So does a name + // too long for the column: one bad entry must not fail the upsert and end the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored_display_name.clone() + } + Some(from_cache) => from_cache.clone(), + None => stored_display_name.clone(), + }; + + if let Some((existing_schema, existing_desc, _, _, _)) = existing { + if existing_schema == &rt.schema + && existing_desc == &rt.description + && stored_extension == format_extension + && stored_display_name == display_name + { skipped_count += 1; continue; } @@ -530,14 +628,21 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // Insert or update resource type sqlx::query( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // `format_extension` is resolved above rather than coalesced here: a + // COALESCE could never clear one, so a hub that dropped an extension + // would leave the stale value behind forever. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, + display_name = EXCLUDED.display_name, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) + .bind(&format_extension) + .bind(&display_name) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; @@ -1261,10 +1366,12 @@ Windmill Community Edition {GIT_VERSION} #[cfg(all(feature = "tantivy", feature = "parquet"))] let log_indexer_f = { let log_indexer_rx = killpill_rx.resubscribe(); - let log_index_writer2 = log_index_writer.clone(); + // Moved, not cloned: sealing a chunk takes sole ownership of its + // tantivy writer, which a second live handle would silently prevent. + let moved_log_index_writer = log_index_writer; async { if let Some(db) = conn.as_sql() { - if let Some(log_index_writer) = log_index_writer2 { + if let Some(log_index_writer) = moved_log_index_writer { windmill_indexer::service_logs_oss::run_indexer( db.clone(), log_index_writer, @@ -1661,7 +1768,7 @@ Windmill Community Edition {GIT_VERSION} } else { tracing::info!("Nothing to do, exiting."); } - send_current_log_file_to_object_store(&conn, &hostname, &mode).await; + flush_pending_log_files_to_object_store(&conn, &hostname, &mode).await; if let Some(db) = conn.as_sql() { tracing::info!("Exiting connection pool"); @@ -1855,6 +1962,17 @@ async fn process_notify_event( ); windmill_api::auth::invalidate_token_from_cache(payload); } + "notify_user_email_change" => { + // `:`, or `*:` from a `password` change, which + // knows the name but no workspace. Workspace ids can't contain ':'. + if let Some(username) = payload.strip_prefix("*:") { + tracing::info!("Superadmin identity change detected, invalidating: {username}"); + windmill_common::users::invalidate_email_cache_for_username(username); + } else if let Some((workspace_id, username)) = payload.split_once(':') { + tracing::info!("User email change detected, invalidating cache: {payload}"); + windmill_common::users::invalidate_email_cache(workspace_id, username); + } + } "notify_app_policy_change" => { // payload is `:`; workspace ids can't contain ':'. if server_mode { @@ -1886,6 +2004,13 @@ async fn process_notify_event( if let Err(e) = reload_license_key(&db.into()).await { tracing::error!("Failed to reload license key: {e:#}"); } + // The worker-group cache override is Enterprise-only, and nothing else + // re-reads the plan for it: the periodic settings pass runs ahead of + // reload_license_key, so it would see the plan this event just replaced. + #[cfg(feature = "parquet")] + if worker_mode { + crate::monitor::reload_cache_object_store_override_with_retry(db).await; + } } DEFAULT_TAGS_PER_WORKSPACE_SETTING => { if let Err(e) = load_tag_per_workspace_enabled(db).await { @@ -1952,6 +2077,12 @@ async fn process_notify_event( } TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await, RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await, + SERVICE_LOG_RETENTION_SECS_SETTING => { + reload_service_log_retention_secs_setting(conn).await + } + OTEL_TRACES_RETENTION_SECS_SETTING => { + reload_otel_traces_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); @@ -2035,6 +2166,9 @@ async fn process_notify_event( DISABLE_PASSWORD_LOGIN_SETTING => { load_disable_password_login(db).await; } + MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING => { + load_mcp_disable_token_query_param(db).await; + } EXPOSE_METRICS_SETTING => { tracing::info!("Metrics setting changed, restarting"); spawn_graceful_killpill(tx, db, 30, "metrics setting change", server_mode) @@ -2063,6 +2197,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload app workspaced route setting"); } } + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING => { + if let Err(e) = reload_http_route_default_allowed_origins_setting(db).await { + tracing::error!(error = %e, "Could not reload http route default allowed origins setting"); + } + } HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => { if let Err(e) = reload_http_route_workspaced_route_setting(db).await { tracing::error!(error = %e, "Could not reload http route workspaced route setting"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 00bf26a558..48dd04e089 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -45,7 +45,10 @@ use windmill_common::otel_oss::{ use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::APP_WORKSPACED_ROUTE, - auth::{create_token_for_owner, ephemeral_script_token_label, job_token_expiry_secs}, + auth::{ + create_token_for_owner, ephemeral_script_token_label, job_token_expiry_secs, + TOKEN_EXPIRY_WARNING_DAYS, + }, ee_oss::CriticalErrorChannel, email_oss::send_email_if_possible, error, @@ -62,26 +65,27 @@ use windmill_common::{ FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, + MCP_DISABLE_TOKEN_QUERY_PARAM, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WORKSPACE_MAX_QUEUED_JOBS_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, }, indexer::load_indexer_config, jobs::delete_jobs, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, - tracing_init::JSON_FMT, users::truncate_token, utils::{empty_as_none, now_from_db, report_critical_error, Mode, HUB_API_SECRET}, worker::{ @@ -98,16 +102,21 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, - SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, + DEFAULT_OTEL_TRACES_RETENTION_SECS, DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, + JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, + METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, + OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, global_settings::{ - APP_WORKSPACED_ROUTE_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE, - HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + parse_allowed_origins_setting, APP_WORKSPACED_ROUTE_SETTING, + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + }, + queue_metrics::{ + QueueSample, QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX, QUEUE_DELAY_SAME_HEAD_SECS, + QUEUE_METRIC_HEARTBEAT_SECS, QUEUE_METRIC_STALE_SECS, }, }; #[cfg(feature = "parquet")] @@ -281,6 +290,15 @@ pub async fn initial_load( ); if let Some(db) = conn.as_sql() { + // Outside the `server_mode` block below: a `MODE=mcp` process serves the MCP routes + // with `server_mode` false and would otherwise never read this at all. That mode + // joins no monitor loop, so there — as for every global setting, `base_url` + // included — this pass is the only read, and a change lands on restart. + pass.setting( + MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, + false, + |v| async move { apply_mcp_disable_token_query_param(v) }, + ); pass.setting(DEFAULT_TAGS_PER_WORKSPACE_SETTING, false, |v| async move { apply_tag_per_workspace_enabled(v) }); @@ -355,7 +373,9 @@ pub async fn initial_load( ) } }); - pass.action(windmill_common::min_version::store_min_keep_alive_version(db)); + pass.action(windmill_common::min_version::store_min_keep_alive_version( + db, + )); pass.setting( windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING, false, @@ -397,6 +417,8 @@ pub async fn initial_load( additional_python_paths: None, pip_local_dependencies: None, native_mode, + // an agent worker never reads its group's config, only its token + object_store_cache_config: None, })); } } @@ -421,6 +443,18 @@ pub async fn initial_load( pass.setting(APP_WORKSPACED_ROUTE_SETTING, false, |v| async move { apply_app_workspaced_route_setting(v) }); + pass.setting( + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + false, + |v| async move { + if let Err(e) = apply_http_route_default_allowed_origins_setting(v) { + tracing::error!( + "Error reloading http route default allowed origins: {:?}", + e + ) + } + }, + ); pass.setting( HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, false, @@ -477,6 +511,19 @@ pub async fn initial_load( |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, ); + // Outside the `server_mode` guard below: every mode reads this. A worker registers its + // rotated log files against the cutoff, and a dedicated indexer trims the search index to a + // window derived from it — neither is a server. + pass.setting(SERVICE_LOG_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_service_log_retention_secs(parse_setting_value::( + v, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + )) + }); + if server_mode { pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { JOB_RETENTION_SECS.store( @@ -503,6 +550,15 @@ pub async fn initial_load( Ordering::Relaxed, ) }); + pass.setting(OTEL_TRACES_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_otel_traces_retention_secs(parse_setting_value::( + v, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + )) + }); pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { STORE_AUDIT_LOGS_S3.store( parse_setting_value::( @@ -688,7 +744,6 @@ pub async fn initial_load( pass.run(conn).await; } - pub fn apply_metrics_enabled(value: Option) { if let Some(serde_json::Value::Bool(t)) = value { METRICS_ENABLED.store(t, Ordering::Relaxed) @@ -1045,8 +1100,8 @@ pub fn apply_fork_workspace_tag_append_fork_suffix(value: Option error::Result<()> { - let v = - load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await?; + let v = load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true) + .await?; apply_critical_alert_mute_ui_setting(v); Ok(()) } @@ -1222,32 +1277,60 @@ async fn sleep_until_next_minute_start_plus_one_s() { } use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; -async fn find_two_highest_files(hostname: &str) -> (Option, Option) { + +/// The minutely rolling appender names each file `.log.<%Y-%m-%d-%H-%M>`; +/// anything else in the directory is not a rotated log file. +fn parse_log_file_ts(file_name: &str) -> Option { + NaiveDateTime::parse_from_str( + file_name.rsplit('.').next()?, + windmill_common::tracing_init::LOG_TIMESTAMP_FMT, + ) + .ok() +} + +/// Oldest first. Readdir order is filesystem-dependent — tmpfs hands back the +/// newest entry first, ext4 hashes the names — so the listing has to be sorted +/// before anything picks a file out of it. +fn sorted_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = file_names + .filter_map(|name| parse_log_file_ts(&name).map(|ts| (ts, name))) + .collect::>(); + files.sort(); + files +} + +/// Every log file but the newest one: that one is still being appended to, every +/// older one is final. +fn rotated_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = sorted_log_files(file_names); + files.pop(); + files +} + +async fn read_log_file_names(hostname: &str) -> Vec { let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); - let rd_dir = tokio::fs::read_dir(log_dir).await; - if let Ok(mut log_files) = rd_dir { - let mut highest_file: Option = None; - let mut second_highest_file: Option = None; - while let Ok(Some(file)) = log_files.next_entry().await { - let file_name = file - .file_name() - .to_str() - .map(|x| x.to_string()) - .unwrap_or_default(); - if file_name > highest_file.clone().unwrap_or_default() { - second_highest_file = highest_file; - highest_file = Some(file_name); - } + let mut rd_dir = match tokio::fs::read_dir(&log_dir).await { + Ok(rd_dir) => rd_dir, + Err(e) => { + tracing::error!("Error reading log files: {}, {:#?}", log_dir, e); + return vec![]; + } + }; + let mut file_names = vec![]; + while let Ok(Some(file)) = rd_dir.next_entry().await { + if let Some(file_name) = file.file_name().to_str() { + file_names.push(file_name.to_string()); } - (highest_file, second_highest_file) - } else { - tracing::error!( - "Error reading log files: {}, {:#?}", - *TMP_WINDMILL_LOGS_SERVICE, - rd_dir.unwrap_err() - ); - (None, None) } + file_names +} + +async fn list_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + sorted_log_files(read_log_file_names(hostname).await.into_iter()) +} + +async fn list_rotated_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + rotated_log_files(read_log_file_names(hostname).await.into_iter()) } fn get_worker_group(mode: &Mode) -> Option { @@ -1267,133 +1350,188 @@ pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + init_last_log_file_sent(&conn, &hostname).await; sleep_until_next_minute_start_plus_one_s().await; loop { interval.tick().await; - let (_, snd_highest_file) = find_two_highest_files(&hostname).await; - send_log_file_to_object_store( - &hostname, - &mode, - &worker_group, - &conn, - snd_highest_file, - false, - ) - .await; + let files = list_rotated_log_files(&hostname).await; + send_log_files_to_object_store(&hostname, &mode, &worker_group, &conn, files).await; } }); } -pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) { - tracing::info!("Sending current log file to object store"); - let (highest_file, _) = find_two_highest_files(hostname).await; +pub async fn flush_pending_log_files_to_object_store( + conn: &Connection, + hostname: &str, + mode: &Mode, +) { + tracing::info!("Sending pending log files to object store"); let worker_group = get_worker_group(&mode); - send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await; -} - -fn get_now_and_str() -> (NaiveDateTime, String) { - let ts = Utc::now().naive_utc(); - ( - ts, - ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) - .to_string(), - ) + // Nothing rotates after this, so the file still being appended to is registered + // here, along with any rotated one the loop had not reached yet. Bounded like the + // pool close that follows: a backlog against a slow object store would otherwise + // hold the process past its termination grace period. Whatever is left over is + // registered by the next run's catch-up. + let flush = async { + let files = list_log_files(hostname).await; + send_log_files_to_object_store(hostname, mode, &worker_group, conn, files).await; + }; + if timeout(Duration::from_secs(15), flush).await.is_err() { + tracing::warn!("Could not send all pending log files in time (15s). Exiting anyway."); + } } lazy_static::lazy_static! { static ref LAST_LOG_FILE_SENT: Arc>> = Arc::new(Mutex::new(None)); + /// Serializes the periodic uploader against the shutdown flush. The uploader is a + /// detached task that keeps ticking while the flush runs and both walk the same + /// files, so without this both can clear the watermark for one file and count its + /// lines twice through the additive upsert. + static ref SENDING_LOG_FILES: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); } +fn last_log_file_sent() -> Option { + LAST_LOG_FILE_SENT.lock().ok().and_then(|ts| *ts) +} + +/// Resume from what this host already registered, so a previous run's leftovers reach +/// the object store rather than being dropped. Their line counts come out zero, this +/// run having counted none of them, which only flattens their bars in the UI. +/// +/// The newest registered minute is left out on purpose: the shutdown flush registers +/// the file that was still open and the appender reopens that minute in append mode, +/// so a restart inside it would otherwise strand everything written afterwards. +/// +/// A row rewritten this way restores the object and sums the counters, but it keeps the +/// `indexed_at` it already had, so one the indexers have taken is not offered again and +/// the lines added by the rewrite stay out of search. +async fn init_last_log_file_sent(conn: &Connection, hostname: &str) { + let Some(db) = conn.as_sql() else { + return; + }; + match sqlx::query_scalar!( + "SELECT max(log_ts) FROM log_file + WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + hostname + ) + .fetch_one(db) + .await + { + Ok(Some(ts)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error initializing last log file sent: {:?}", e); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error loading last log file sent: {:?}", e), + } +} + +async fn send_log_files_to_object_store( + hostname: &str, + mode: &Mode, + worker_group: &Option, + conn: &Connection, + files: Vec<(NaiveDateTime, String)>, +) { + let _guard = SENDING_LOG_FILES.lock().await; + let retention_cutoff = Utc::now().naive_utc() + - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); + for (ts, file_name) in files { + if last_log_file_sent().is_some_and(|last| last >= ts) { + continue; + } + // A run coming back from a long outage still finds its predecessor's files on + // disk. Registering one past the retention cutoff inserts a row + // `delete_expired_items` drops on its next pass, once the indexers have already + // paid to parse it. + if ts < retention_cutoff { + continue; + } + // Stop at the first failure rather than moving on, so a file is never + // registered before an older one that has not made it to the store yet. + // The indexers do not depend on that ordering — every row is offered until + // it is marked — but a gap here would still be visible while it lasts. + if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await + { + break; + } + } +} + +/// Returns whether the file ended up registered in `log_file`. async fn send_log_file_to_object_store( hostname: &str, mode: &Mode, worker_group: &Option, conn: &Connection, - snd_highest_file: Option, - use_now: bool, -) { - if let Some(highest_file) = snd_highest_file { - //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm - let (ts, ts_str) = if use_now { - get_now_and_str() - } else { - highest_file - .split(".") - .last() - .and_then(|x| { - NaiveDateTime::parse_from_str( - x, - windmill_common::tracing_init::LOG_TIMESTAMP_FMT, - ) - .ok() - .map(|y| (y, x.to_string())) - }) - .unwrap_or_else(get_now_and_str) - }; + file_name: &str, + ts: NaiveDateTime, +) -> bool { + #[cfg(feature = "parquet")] + if let Some(s3_client) = windmill_object_store::get_object_store().await { + let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) + .join(hostname) + .join(file_name); - let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| { - last_log_file_sent - .map(|last_log_file_sent| last_log_file_sent >= ts) - .unwrap_or(false) - }); - - if exists.unwrap_or(false) { - return; - } - - #[cfg(feature = "parquet")] - let s3_client = windmill_object_store::get_object_store().await; - #[cfg(feature = "parquet")] - if let Some(s3_client) = s3_client { - let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) - .join(hostname) - .join(&highest_file); - - //read file as byte stream - let bytes = tokio::fs::read(&path).await; - if let Err(e) = bytes { + //read file as byte stream + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(e) => { tracing::error!("Error reading log file: {:?}", e); - return; + return false; } - let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( - "{}{hostname}/{highest_file}", - windmill_common::tracing_init::LOGS_SERVICE - )); - if let Err(e) = path { + }; + let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( + "{}{hostname}/{file_name}", + windmill_common::tracing_init::LOGS_SERVICE + )); + let path = match path { + Ok(path) => path, + Err(e) => { tracing::error!("Error creating log file path: {:?}", e); - return; - } - if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await { - tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + }; + if let Err(e) = s3_client.put(&path, bytes.into()).await { + tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + } - let (ok_lines, err_lines) = read_log_counters(ts_str); + let ts_str = ts + .format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .to_string(); + let (ok_lines, err_lines) = read_log_counters(ts_str); - if let Some(db) = conn.as_sql() { - match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) - VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) - ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", - hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) - .execute(db)).await { - Ok(Ok(_)) => { - if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { - last_log_file_sent.replace(ts); - }) { - tracing::error!("Error updating last log file sent: {:?}", e); - } - tracing::info!("Log file sent: {}", highest_file); - } - Ok(Err(e)) => { - tracing::error!("Error inserting log file: {:?}", e); - } - Err(e) => { - tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); - } + let Some(db) = conn.as_sql() else { + // not sending log file to object store in agent mode + return false; + }; + + match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) + VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) + ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + hostname, mode.to_string(), worker_group.clone(), ts, file_name, ok_lines as i64, err_lines as i64, true) + .execute(db)).await { + Ok(Ok(_)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error updating last log file sent: {:?}", e); } - } else { - // tracing::warn!("Not sending log file to object store in agent mode"); - () + tracing::info!("Log file sent: {}", file_name); + true + } + Ok(Err(e)) => { + tracing::error!("Error inserting log file: {:?}", e); + false + } + Err(e) => { + tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); + false } } } @@ -1490,6 +1628,23 @@ pub fn apply_disable_password_login(value: Option) { }; } +pub async fn load_mcp_disable_token_query_param(db: &DB) { + match load_value_from_global_settings(db, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING).await { + Ok(v) => apply_mcp_disable_token_query_param(v), + Err(e) => tracing::error!("Error loading mcp_disable_token_query_param setting: {e:#}"), + }; +} + +pub fn apply_mcp_disable_token_query_param(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { + MCP_DISABLE_TOKEN_QUERY_PARAM.store(t, Ordering::Relaxed) + } + None => MCP_DISABLE_TOKEN_QUERY_PARAM.store(false, Ordering::Relaxed), + _ => (), + }; +} + struct LogFile { file_path: String, hostname: String, @@ -1581,6 +1736,64 @@ pub async fn trim_resource_versions(db: &DB) -> () { } } +/// Matches the batch the settings-page cleanup uses for the same table. +const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; +/// Batches per pass. `monitor_db` runs under a 600s timeout that cancels every maintenance +/// future in the same `join!` and reports a critical error, so a large backlog has to drain +/// across ticks rather than inside one, the way the neighbouring sweeps already do. +const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; + +/// One span per HTTP request made from a job script, so the table grows far faster than the +/// job table it is keyed against; batched for the same reason the service log sweep is. +const OTEL_TRACES_DELETE_BATCH: i64 = 10_000; +const OTEL_TRACES_DELETE_MAX_BATCHES: usize = 10; + +/// Delete HTTP request tracing spans older than `retention_secs`, returning how many went. +/// +/// `retention_secs` is a parameter rather than a read of the process-wide setting so a test can +/// pin a window without writing state the other tests in this binary run against concurrently. +async fn delete_expired_otel_traces(db: &DB, retention_secs: i64) -> u64 { + // `start_time_unix_nano` is the proto field stored verbatim, so the cutoff is built in that + // unit rather than compared against `now()`. Truncating the epoch to whole seconds first + // keeps the multiplication inside `bigint`. + // + // Batched on `ctid`, not on the `(trace_id, span_id)` primary key: with the key the planner + // hashes the LIMITed subquery and Seq Scans the whole table to probe it, which at the size + // this table reaches is the cost the batching exists to avoid. `ctid` plans as a Tid Scan, so + // each batch touches only the rows it deletes. Safe because the subquery and the delete share + // one snapshot, and spans are never updated after insert. + let mut deleted = 0; + for _ in 0..OTEL_TRACES_DELETE_MAX_BATCHES { + let batch = sqlx::query!( + "DELETE FROM otel_traces WHERE ctid IN ( + SELECT ctid FROM otel_traces + WHERE start_time_unix_nano < EXTRACT( + EPOCH FROM now() - ($1::bigint::text || ' s')::interval + )::bigint * 1000000000 + LIMIT $2 + )", + retention_secs, + OTEL_TRACES_DELETE_BATCH, + ) + .execute(db) + .await; + + match batch { + Ok(res) => { + deleted += res.rows_affected(); + if (res.rows_affected() as i64) < OTEL_TRACES_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting expired otel trace spans: {:?}", e); + break; + } + } + } + deleted +} + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1602,6 +1815,23 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), } + let expired_login_links_r: std::result::Result, _> = + // Expired rows stay a day so an open still reports "expired" rather than "invalid". + sqlx::query_scalar( + "DELETE FROM login_link WHERE expiration <= now() - interval '1 day' RETURNING token_hash", + ) + .fetch_all(db) + .await; + + match expired_login_links_r { + Ok(hashes) => { + if !hashes.is_empty() { + tracing::info!("deleted {} expired login links", hashes.len()) + } + } + Err(e) => tracing::error!("Error deleting login links: {}", e.to_string()), + } + let pip_resolution_r = sqlx::query_scalar!( "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash", ) @@ -1663,37 +1893,65 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()), } - match sqlx::query_as!( - LogFile, - "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, - ) - .fetch_all(db) - .await - { - Ok(log_files_to_delete) => { + // Batched: every process rotates a log file a minute, so lowering the retention makes one + // ordinary setting change expire millions of rows at once. An unbounded `DELETE ... + // RETURNING` would materialize all of them, and their deletion futures, in this one tick. + for _ in 0..SERVICE_LOG_DELETE_MAX_BATCHES { + let batch = sqlx::query_as!( + LogFile, + "DELETE FROM log_file WHERE (hostname, log_ts) IN ( + SELECT hostname, log_ts FROM log_file + WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval + LIMIT $2 + ) RETURNING file_path, hostname", + windmill_common::service_log_retention_secs(), + SERVICE_LOG_DELETE_BATCH, + ) + .fetch_all(db) + .await; + + match batch { + Ok(log_files_to_delete) => { + if log_files_to_delete.is_empty() { + break; + } + let n = log_files_to_delete.len(); let paths = log_files_to_delete .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; - + delete_log_files_from_disk_and_store( + paths, + &*TMP_WINDMILL_LOGS_SERVICE, + windmill_common::tracing_init::LOGS_SERVICE, + ) + .await; + if (n as i64) < SERVICE_LOG_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting log file: {:?}", e); + break; + } } - Err(e) => tracing::error!("Error deleting log file: {:?}", e), } - let audit_retention_days = audit_log_retention_days().await; - let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; + let deleted_spans = + delete_expired_otel_traces(db, windmill_common::otel_traces_retention_secs()).await; + if deleted_spans > 0 { + tracing::info!("deleted {} expired otel trace spans", deleted_spans); + } - // Clean up old (non-partitioned) audit table — will eventually be empty and dropped - if let Err(e) = sqlx::query_scalar!( - "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval", - audit_retention_secs, + if let Err(e) = sqlx::query!( + "DELETE FROM ai_shared_artifact + WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + windmill_common::ai_shared_artifact_retention_secs(), ) - .fetch_all(db) + .execute(db) .await { - tracing::error!("Error deleting audit log: {:?}", e); + tracing::error!("Error deleting expired shared AI artifacts: {:?}", e); } if let Err(e) = sqlx::query_scalar!( @@ -1744,6 +2002,15 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting old feature_usage rows: {e}"); } + // Guest sign-ins, kept a month longer than the seat window they feed so a late + // telemetry send still sees a whole month. + if let Err(e) = sqlx::query!("DELETE FROM guest_activity WHERE day < CURRENT_DATE - 60") + .execute(db) + .await + { + tracing::error!("Error deleting old guest_activity rows: {e}"); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) @@ -1948,7 +2215,7 @@ async fn cleanup_scheduled_job_deletions(db: &Pool) { } pub async fn check_expiring_tokens(db: &DB) { - // Find tokens expiring within 7 days that still have a pending notification row. + // Find tokens expiring within the warning window that still have a pending notification row. // The notification table stores token_hash (not plaintext) so the join works // even after the hash migration makes token.token nullable. let expiring_tokens_r = sqlx::query_as!( @@ -1957,8 +2224,9 @@ pub async fn check_expiring_tokens(db: &DB) { USING token t WHERE n.token_hash = t.token_hash AND n.expiration > now() - AND n.expiration <= now() + interval '7 days' + AND n.expiration <= now() + make_interval(days => $1) RETURNING t.token_prefix, t.label, t.email, t.workspace_id", + TOKEN_EXPIRY_WARNING_DAYS, ) .fetch_all(db) .await; @@ -2530,7 +2798,6 @@ pub async fn reload_timeout_wait_result_setting(conn: &Connection) { .await; } - pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2621,7 +2888,6 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { .await; } - pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2729,7 +2995,6 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } - pub async fn reload_workspace_registries_setting(conn: &Connection) { match load_value_from_global_settings_with_conn( conn, @@ -2786,6 +3051,36 @@ pub async fn reload_retention_period_setting(conn: &Connection) { } } +pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_service_log_retention_secs(v), + Err(e) => tracing::error!("Error reloading service log retention period: {:?}", e), + } +} + +pub async fn reload_otel_traces_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_otel_traces_retention_secs(v), + Err(e) => tracing::error!("Error reloading otel traces retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, @@ -2953,7 +3248,6 @@ pub async fn apply_job_isolation_setting(value: Option) { } } - async fn resolve_license_key_value(conn: &Connection, quiet: bool) -> anyhow::Result { let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) .await @@ -3248,7 +3542,10 @@ impl<'a> SettingsPass<'a> { // on compile-time defaults until the next full reload. Only the single-query transport // can fail this way; over HTTP the batch already is the per-setting read. if matches!(conn, Connection::Sql(_)) && values.is_empty() && !names.is_empty() { - tracing::warn!("Falling back to per-setting reads for {} settings", names.len()); + tracing::warn!( + "Falling back to per-setting reads for {} settings", + names.len() + ); values = fetch_settings_individually(conn, &names).await; } for (name, http) in &declared { @@ -3640,7 +3937,6 @@ pub fn parse_setting_value( value } - #[cfg(feature = "prometheus")] pub async fn monitor_pool(db: &DB) { if METRICS_ENABLED.load(Ordering::Relaxed) { @@ -4094,6 +4390,41 @@ pub async fn monitor_db( } }; + // Re-check what each git-sync repository's own credential says about its expiry, + // and rotate the ones close to it. Every ~40 min: the values move over days, and + // `should_run` counts iterations in a u8. Spawned rather than joined: the join + // below is cancelled at its deadline, which a long sweep would reach, and a + // rotation cut off between GitLab issuing a token and Windmill storing it + // loses the token family. Detached, only process shutdown can cut it off, + // which a rotation almost never coincides with. The pass's advisory lock + // keeps a slow one from overlapping the next. + let git_credential_maintenance_f = async { + #[cfg(all(feature = "enterprise", feature = "private"))] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { maintain_git_credentials(&db).await }); + } + } + }; + + // Delete the AI session backups older than their workspace's retention. Every ~40 min + // (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention + // counts in days. Spawned for the same reason as the credential maintenance above, a + // sweep of many sessions outlasting the join's deadline; the sweep's own advisory lock + // keeps one server at a time at it. + let ai_session_retention_f = async { + #[cfg(feature = "parquet")] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn( + async move { windmill_api::sweep_expired_ai_session_backups(&db).await }, + ); + } + } + }; + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). // Enterprise feature: the active `// freshness` backstop lives in // windmill-queue's `freshness_watchdog` (`private`); OSS gets a no-op stub. @@ -4148,6 +4479,8 @@ pub async fn monitor_db( export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, git_auto_pull_f, + git_credential_maintenance_f, + ai_session_retention_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, ); @@ -4408,17 +4741,11 @@ const GIT_AUTO_PULL_LOCK_ID: i64 = 737_483_921; /// Poll every git-sync repository with auto-pull enabled and enqueue a pull when /// the tracked branch has new commits (repo → Windmill direction). /// -/// Runs on a single replica at a time (advisory lock) and only on -/// Enterprise-licensed instances. Detection is `git ls-remote`; GitHub-App -/// repositories are skipped here and sync via webhooks instead (phase 2). +/// Runs on a single replica at a time (advisory lock). Detection is +/// `git ls-remote`; GitHub-App repositories are skipped here and sync via +/// webhooks instead (phase 2). #[cfg(feature = "private")] pub async fn poll_git_auto_pull(db: &Pool) { - use windmill_common::ee_oss::{get_license_plan, LicensePlan}; - - if !matches!(get_license_plan().await, LicensePlan::Enterprise) { - return; - } - let mut lock_conn = match db.acquire().await { Ok(c) => c, Err(e) => { @@ -4453,6 +4780,14 @@ pub async fn poll_git_auto_pull(db: &Pool) { { tracing::error!("git auto-pull: advisory unlock failed: {e:#}"); } + + // Backstop for the "Windmill CI tests" checks: retry a failed GitHub create or + // delivery, conclude checks whose tests settled, time out stuck ones, prune old + // rows. Detached and outside the advisory lock: its writes are guarded (claimed + // conclude, greatest-id upsert), it is single-flight, and its GitHub calls must not + // count against the monitor pass's budget. + let db = db.clone(); + tokio::spawn(async move { windmill_git_sync::sweep_ci_test_checks(&db).await }); } #[cfg(feature = "private")] @@ -4470,6 +4805,150 @@ lazy_static::lazy_static! { #[cfg(feature = "private")] const AUTO_PULL_POLL_SLACK_S: i64 = 30; +/// Advisory lock id ensuring only one server replica maintains git credentials at +/// a time (adjacent to GIT_AUTO_PULL_LOCK_ID). +#[cfg(all(feature = "enterprise", feature = "private"))] +const GIT_CREDENTIAL_LOCK_ID: i64 = 737_483_923; + +/// Refresh every git-sync repository's credential status and rotate the ones near +/// expiry, so a token dies visibly (and usually not at all) rather than taking +/// sync down on its expiry date. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials(db: &Pool) { + // Transaction-scoped advisory lock, as for the schedule reconcile above: a + // session lock on a pooled connection would ride back into the pool still + // held if the sweep died before unlocking, and wedge the pass on every + // replica until a restart. The transaction only owns the lock; the sweep + // commits each status and each rotated token on its own as it goes. + let mut lock_tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("git credentials: failed to begin lock tx: {e:#}"); + return; + } + }; + // The transaction stays idle while the sweep talks to git hosts, and the + // pool's ten-minute idle-in-transaction timeout would end it, lock included, + // partway through a sweep over enough slow hosts. Lifted for this + // transaction only; it dies with the connection either way. + if let Err(e) = sqlx::query("SET LOCAL idle_in_transaction_session_timeout = 0") + .execute(&mut *lock_tx) + .await + { + tracing::error!("git credentials: failed to lift the idle timeout: {e:#}"); + return; + } + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(GIT_CREDENTIAL_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("git credentials: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + + if let Err(e) = maintain_git_credentials_inner(db).await { + tracing::error!("git credentials: maintenance error: {e:#}"); + } + drop(lock_tx); +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials_inner(db: &Pool) -> error::Result<()> { + use windmill_common::workspaces::WorkspaceGitSyncSettings; + + // Same deleted/archived exclusion as the auto-pull poller: a dead workspace's + // settings row survives, and rotating a token for one would be pure damage. + let rows = sqlx::query!( + r#"SELECT ws.workspace_id, ws.git_sync + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE NOT w.deleted + AND ws.git_sync IS NOT NULL + AND jsonb_typeof(ws.git_sync->'repositories') = 'array'"# + ) + .fetch_all(db) + .await?; + + for row in rows { + let Some(git_sync) = row.git_sync else { + continue; + }; + let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + "git credentials: invalid git_sync settings for workspace {}: {e}", + row.workspace_id + ); + continue; + } + }; + + for repo in settings.repositories.iter() { + let path = &repo.git_repo_resource_path; + // This refreshes and records the status on every repository it looks at, + // rotating only the ones near expiry, so it is the whole maintenance pass + // rather than just the rotation half. + if let Err(e) = windmill_common::git_sync_ee::rotate_git_credential_if_due( + db, + &row.workspace_id, + path, + ) + .await + { + tracing::error!( + "git credentials: maintenance failed for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + + // A repository that wants webhook delivery but holds no hook never + // gets one otherwise: the reconcile runs on a settings save, so a + // credential that was unusable when the hook should have been created + // would leave it missing until an admin saved again. Checking stored + // state costs nothing, and only the repositories actually missing a + // hook reach the host. + use windmill_common::workspaces::AutoPullMode; + // Also when a hook exists but carries a warning: a save during a GitLab + // outage keeps the hook and records why it could not be confirmed, and + // that warning is only cleared by a reconcile that confirms it again. + // Only repositories with a checked credential of their own: for + // everything else a settings save stays the one place hooks are + // reconciled, so an App repository or a plain remote is never touched + // here, and a recorded delivery mode nobody saved is never normalized. + let needs_hook = repo.credential.is_some() + && repo.auto_pull.as_ref().is_some_and(|a| { + a.enabled + && matches!(a.mode, AutoPullMode::Auto | AutoPullMode::Webhook) + && (a.webhook_id.is_none() || a.webhook_error.is_some()) + }); + if needs_hook { + let mut repo = repo.clone(); + if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook( + db, + &row.workspace_id, + &mut repo, + ) + .await + { + tracing::warn!( + "git credentials: could not reconcile the webhook for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + } + } + } + Ok(()) +} + #[cfg(feature = "private")] async fn poll_git_auto_pull_inner(db: &Pool) -> error::Result<()> { use windmill_common::workspaces::{AutoPullMode, WorkspaceGitSyncSettings}; @@ -4709,161 +5188,309 @@ async fn poll_git_fork_branches( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces") .execute(db) .await?; Ok(()) } -pub async fn expose_queue_metrics(db: &Pool) { - let last_check = sqlx::query_scalar!( - "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1" - ) - .fetch_optional(db) - .await - .unwrap_or(Some(chrono::Utc::now())); +/// Shortest spacing between two stored samples of the same queue metric, so a tag whose +/// value moves on every monitor round still writes at most one row per interval. Also how +/// often each server samples the queue when no Prometheus or OTel gauge needs it sooner. +const QUEUE_METRIC_MIN_INTERVAL_SECS: f64 = 25.0; +/// A held delay hovers while the head keeps changing, so an exact-value comparison would rarely +/// dedup it. Only a move the chart would actually render is stored. +const QUEUE_DELAY_TOLERANCE: f64 = 0.1; - let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); - let save_metrics = last_check - .map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25)) - .unwrap_or(true); +/// Append the queue metrics the drawer at `GET /workers/queue_metrics_series` charts, skipping +/// any sample that repeats what is already stored. +/// +/// Only tags with a backlog appear in `queue_stats`, and an arbitrary `?tag=` nobody serves +/// stays backlogged forever, so writing every round would repeat the same pair of rows for +/// the whole 14-day retention. Each metric is written when the value it draws moves, once per +/// heartbeat while it holds, and once more (as a zero) when the tag drains. Gaps therefore +/// mean "unchanged since the last row", which is what the chart interpolates. A delay whose +/// head job stays put is stored as that job's wait start, which the chart draws climbing, so it +/// never moves away from what is stored either. +async fn save_queue_metrics( + db: &Pool, + queue_stats: &std::collections::HashMap, +) { + let sampled_ids = queue_stats + .keys() + .flat_map(|tag| { + [ + format!("{QUEUE_COUNT_PREFIX}{tag}"), + format!("{QUEUE_DELAY_PREFIX}{tag}"), + ] + }) + .collect::>(); - if metrics_enabled || save_metrics || OTEL_METRICS_ENABLED.load(Ordering::Relaxed) { - let queue_counts = windmill_common::queue::get_queue_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - (*QUEUE_COUNT).with_label_values(&[q]).set(0); - } - } + // Last stored sample of every metric that either has a backlog now or was written + // recently enough to still be believed backlogged. Bounding the lookup by the stale window + // keeps it cheap at any `metrics` size; a per-id `ORDER BY created_at DESC LIMIT 1` does + // not, since the planner may serve it from `metrics_sort_idx` and walk the whole table. + let last_samples = match sqlx::query!( + "SELECT COALESCE(c.id, r.id) AS \"id!\", r.value AS \"value?\", + EXTRACT(EPOCH FROM r.created_at)::double precision AS \"at?\", + EXTRACT(EPOCH FROM now() - r.created_at)::double precision AS \"age?\" + FROM unnest($1::text[]) AS c(id) + FULL JOIN ( + SELECT DISTINCT ON (id) id, value, created_at + FROM metrics + WHERE id LIKE 'queue_%' AND created_at > now() - make_interval(secs => $2) + ORDER BY id, created_at DESC + ) r ON r.id = c.id", + &sampled_ids[..], + QUEUE_METRIC_STALE_SECS, + ) + .fetch_all(db) + .await + { + Ok(rows) => rows, + Err(e) => { + tracing::error!("Failed to read last queue metrics samples: {e:#}"); + return; } + }; - let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let mut ids = vec![]; + let mut values = vec![]; + // The wait start of the head of each held delay, whose value the INSERT computes. + let mut held_heads: Vec> = vec![]; + for row in last_samples { + let Some((prefix, tag)) = [QUEUE_COUNT_PREFIX, QUEUE_DELAY_PREFIX] + .into_iter() + .find_map(|p| row.id.strip_prefix(p).map(|tag| (p, tag))) + else { + continue; + }; + // A stored value that cannot be read cannot be compared, so the next reading is kept. + let last = row + .value + .as_ref() + .and_then(QueueSample::parse) + .zip(row.at) + .zip(row.age) + .map(|((sample, at), age)| (sample, at, age)); + let stat = queue_stats.get(tag); + let current = stat.map(|stat| { + if prefix == QUEUE_COUNT_PREFIX { + stat.count as f64 + } else { + stat.delay + } + }); - if otel_enabled { - for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { - if queue_counts.get(q).is_none() { - otel_set_queue_count(q, 0); + let next_delay = stat + .filter(|_| prefix == QUEUE_DELAY_PREFIX) + .map(|stat| delay_sample(last.map(|(sample, at, _)| sample.head_since(at)), stat)); + let drawn_now = last.map(|(sample, at, age)| (sample.value_at(at + age), age)); + let redraws = last + .zip(next_delay) + .is_some_and(|((sample, ..), next)| redraws(sample, next)); + if should_store(prefix, drawn_now, current, redraws) { + let (value, held_head) = match (stat, next_delay) { + (None, _) => (serde_json::json!(0), None), + (Some(stat), None) => (serde_json::json!(stat.count), None), + (Some(stat), Some(QueueSample::Held(_))) => { + (serde_json::Value::Null, Some(stat.head_since)) } - } - } - - #[allow(unused_mut)] - let mut tags_to_watch = vec![]; - #[allow(unused_mut)] - let mut otel_tags_to_watch = vec![]; - for q in queue_counts { - let count = q.1; - let tag = q.0; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); - metric.set(count as i64); - tags_to_watch.push(tag.to_string()); - } - - if otel_enabled { - otel_tags_to_watch.push(tag.to_string()); - } - otel_set_queue_count(&tag, count as i64); - - // save queue_count and delay metrics per tag - if save_metrics { - sqlx::query!( - "INSERT INTO metrics (id, value) VALUES ($1, $2)", - format!("queue_count_{}", tag), - serde_json::json!(count) - ) - .execute(db) - .await - .ok(); - if count > 0 { - sqlx::query!( - "INSERT INTO metrics (id, value) - VALUES ($1, to_jsonb(( - SELECT EXTRACT(EPOCH FROM now() - scheduled_for) - FROM v2_job_queue - WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval - ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1 - )))", - format!("queue_delay_{}", tag), - tag - ) - .execute(db) - .await - .ok(); - } - } - } - if metrics_enabled { - let mut w = QUEUE_COUNT_TAGS.write().await; - *w = tags_to_watch; - } - if otel_enabled { - let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; - *w = otel_tags_to_watch; - } - - // Single DB query for running counts, shared by Prometheus and OTel - let otel_running = otel_enabled; - #[cfg(feature = "prometheus")] - let need_running_counts = metrics_enabled || otel_running; - #[cfg(not(feature = "prometheus"))] - let need_running_counts = otel_running; - - if need_running_counts { - let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; - - #[cfg(feature = "prometheus")] - if metrics_enabled { - for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); - } - } - } - - if otel_running { - for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { - if queue_running_counts.get(q).is_none() { - otel_set_queue_running_count(q, 0); - } - } - } - - #[allow(unused_mut, unused_variables)] - let mut running_tags_to_watch: Vec = vec![]; - #[allow(unused_mut, unused_variables)] - let mut otel_running_tags_to_watch: Vec = vec![]; - for (tag, count) in &queue_running_counts { - #[cfg(feature = "prometheus")] - if metrics_enabled { - let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); - metric.set(*count as i64); - running_tags_to_watch.push(tag.to_string()); - } - - if otel_running { - otel_set_queue_running_count(tag, *count as i64); - otel_running_tags_to_watch.push(tag.to_string()); - } - } - - #[cfg(feature = "prometheus")] - if metrics_enabled { - let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = running_tags_to_watch; - } - if otel_running { - let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; - *w = otel_running_tags_to_watch; - } + (Some(_), Some(climbing)) => (climbing.to_json(), None), + }; + ids.push(row.id); + values.push(value); + held_heads.push(held_head); } } + if ids.is_empty() { + return; + } + // A held delay is computed from this statement's `now()`, the row's `created_at` too, so + // `created_at - value` is exactly its head's wait start. That is how the next sample tells + // whether the same job is still at the head, within `QUEUE_DELAY_SAME_HEAD_SECS`, which the + // time between reading the queue and this INSERT could otherwise exceed on a busy database. + if let Err(e) = sqlx::query!( + "INSERT INTO metrics (id, value) + SELECT id, COALESCE(to_jsonb(EXTRACT(EPOCH FROM now())::double precision - held_head), value) + FROM unnest($1::text[], $2::jsonb[], $3::double precision[]) AS u(id, value, held_head)", + &ids[..], + &values[..], + &held_heads[..] as &[Option], + ) + .execute(db) + .await + { + tracing::error!("Failed to save queue metrics: {e:#}"); + } +} + +/// What to store for a delay reading, given when the head job of the last stored sample started +/// waiting. The same job still at the head keeps the delay climbing from its wait start, which +/// the chart draws exactly. A head that changed means a moving queue, whose delay hovers and is +/// held; so is a first sample, which cannot tell yet and must not draw a climb that never was. +fn delay_sample( + last_head_since: Option, + stat: &windmill_common::queue::QueueStat, +) -> QueueSample { + match last_head_since { + Some(since) if (since - stat.head_since).abs() < QUEUE_DELAY_SAME_HEAD_SECS => { + QueueSample::Climbing { since: stat.head_since } + } + _ => QueueSample::Held(stat.delay), + } +} + +/// Whether the next delay sample is drawn differently from the last one even at the same value: +/// a climb whose head left would otherwise go on climbing from the old head, and a held delay +/// whose head stayed would stay flat while the wait grows. +fn redraws(last: QueueSample, next: QueueSample) -> bool { + matches!(last, QueueSample::Climbing { .. }) != matches!(next, QueueSample::Climbing { .. }) +} + +/// Whether a reading deserves a row of its own, given the last one stored for that metric: +/// the value it draws now and how many seconds ago it was written. `current` is `None` once +/// the tag has no backlog left; `redraws` is set when the reading must be drawn differently. +fn should_store( + prefix: &str, + last: Option<(f64, f64)>, + current: Option, + redraws: bool, +) -> bool { + let Some((last_value, age)) = last else { + // Nothing comparable within the lookback window: a tag that just backed up needs a + // first sample, one that was already gone needs nothing. + return current.is_some(); + }; + let Some(current) = current else { + // The tag drained. One zero pins where the line drops; after that the metric matches + // and goes quiet, then falls out of the lookback window entirely. + return last_value != 0.0; + }; + if age >= QUEUE_METRIC_HEARTBEAT_SECS { + return true; + } + age >= QUEUE_METRIC_MIN_INTERVAL_SECS + && (redraws + || if prefix == QUEUE_COUNT_PREFIX { + last_value != current + } else { + (current - last_value).abs() > last_value.abs() * QUEUE_DELAY_TOLERANCE + }) +} + +#[cfg(test)] +mod queue_metric_sampling { + use super::*; + + const RECENT: f64 = QUEUE_METRIC_MIN_INTERVAL_SECS + 1.0; + + #[test] + fn a_holding_backlog_writes_only_on_the_heartbeat() { + let held = Some((3.0, RECENT)); + assert!(!should_store(QUEUE_COUNT_PREFIX, held, Some(3.0), false)); + let due = Some((3.0, QUEUE_METRIC_HEARTBEAT_SECS)); + assert!(should_store(QUEUE_COUNT_PREFIX, due, Some(3.0), false)); + // A held delay hovers, so only a move past the tolerance counts as a change. + let delay = Some((100.0, RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, delay, Some(105.0), false)); + assert!(should_store(QUEUE_DELAY_PREFIX, delay, Some(120.0), false)); + } + + #[test] + fn a_drained_tag_writes_one_zero_then_stops() { + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + None, + false + )); + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((0.0, RECENT)), + None, + false + )); + // Including once the heartbeat is due: a tag that is gone stays silent. + let gone = Some((0.0, QUEUE_METRIC_STALE_SECS)); + assert!(!should_store(QUEUE_COUNT_PREFIX, gone, None, false)); + assert!(!should_store(QUEUE_COUNT_PREFIX, None, None, false)); + } + + #[test] + fn a_change_waits_for_the_minimum_interval() { + assert!(!should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, 1.0)), + Some(9.0), + false + )); + assert!(should_store( + QUEUE_COUNT_PREFIX, + Some((3.0, RECENT)), + Some(9.0), + false + )); + // A tag that has just backed up is recorded at once. + assert!(should_store(QUEUE_COUNT_PREFIX, None, Some(9.0), false)); + } + + #[test] + fn a_delay_climbs_while_the_same_job_stays_at_the_head() { + let stat = windmill_common::queue::QueueStat { count: 3, delay: 330.0, head_since: 1000.0 }; + // A held sample written at 1320 saw the same head: it switches to climbing at once, + // although the delay has not moved past the tolerance yet. + let first = QueueSample::Held(320.0); + let climbing = delay_sample(Some(first.head_since(1320.0)), &stat); + assert_eq!(climbing, QueueSample::Climbing { since: 1000.0 }); + assert!(redraws(first, climbing)); + let drawn = Some((first.value_at(1330.0), RECENT)); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(stat.delay), + true + )); + // Stored climbing, it draws the delay exactly: nothing more until the heartbeat. + let drawn = Some((climbing.value_at(1600.0), RECENT)); + assert!(!should_store(QUEUE_DELAY_PREFIX, drawn, Some(600.0), false)); + assert_eq!(delay_sample(None, &stat), QueueSample::Held(330.0)); + } + + #[test] + fn a_climb_whose_head_left_is_held_even_within_the_tolerance() { + // The head waiting since 0 left at 3600 for one queued at 100: 3500s is within 10% of + // the 3600s the climb draws, but kept, the climb would go on from the old head. + let moved = + windmill_common::queue::QueueStat { count: 2, delay: 3500.0, head_since: 100.0 }; + let climbing = QueueSample::Climbing { since: 0.0 }; + let next = delay_sample(Some(climbing.head_since(3000.0)), &moved); + assert_eq!(next, QueueSample::Held(3500.0)); + assert!(redraws(climbing, next)); + let drawn = Some((climbing.value_at(3600.0), RECENT)); + assert!(!should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + false + )); + assert!(should_store( + QUEUE_DELAY_PREFIX, + drawn, + Some(moved.delay), + true + )); + } +} + +/// When this server last sampled the queue into `metrics`, in Unix milliseconds. It only paces +/// how often the queue is scanned for that; whether a sample earns a row is decided from what +/// is already stored. Servers sampling in the same instant can each write it, and the duplicate +/// draws the same. +static LAST_QUEUE_SAMPLE_MS: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +pub async fn expose_queue_metrics(db: &Pool) { // clean queue metrics older than 14 days sqlx::query!( "DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'" @@ -4871,6 +5498,131 @@ pub async fn expose_queue_metrics(db: &Pool) { .execute(db) .await .ok(); + + let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed); + let otel_enabled = OTEL_METRICS_ENABLED.load(Ordering::Relaxed); + let now_ms = chrono::Utc::now().timestamp_millis(); + let save_metrics = now_ms - LAST_QUEUE_SAMPLE_MS.load(Ordering::Relaxed) + >= (QUEUE_METRIC_MIN_INTERVAL_SECS * 1000.0) as i64; + if !(metrics_enabled || otel_enabled || save_metrics) { + return; + } + + // Single DB query for running counts, shared by Prometheus and OTel. It runs ahead of the + // backlog read below, which gives up on the rest of the round when it fails. + let otel_running = otel_enabled; + #[cfg(feature = "prometheus")] + let need_running_counts = metrics_enabled || otel_running; + #[cfg(not(feature = "prometheus"))] + let need_running_counts = otel_running; + + if need_running_counts { + let queue_running_counts = windmill_common::queue::get_queue_running_counts(db).await; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + (*QUEUE_RUNNING_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_running { + for q in OTEL_QUEUE_RUNNING_COUNT_TAGS.read().await.iter() { + if queue_running_counts.get(q).is_none() { + otel_set_queue_running_count(q, 0); + } + } + } + + #[allow(unused_mut, unused_variables)] + let mut running_tags_to_watch: Vec = vec![]; + #[allow(unused_mut, unused_variables)] + let mut otel_running_tags_to_watch: Vec = vec![]; + for (tag, count) in &queue_running_counts { + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_RUNNING_COUNT).with_label_values(&[tag]); + metric.set(*count as i64); + running_tags_to_watch.push(tag.to_string()); + } + + if otel_running { + otel_set_queue_running_count(tag, *count as i64); + otel_running_tags_to_watch.push(tag.to_string()); + } + } + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let mut w = QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = running_tags_to_watch; + } + if otel_running { + let mut w = OTEL_QUEUE_RUNNING_COUNT_TAGS.write().await; + *w = otel_running_tags_to_watch; + } + } + + let queue_stats = match windmill_common::queue::get_queue_stats(db).await { + Ok(queue_stats) => queue_stats, + Err(e) => { + tracing::error!("Failed to read queue stats: {e:#}"); + return; + } + }; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + for q in QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + (*QUEUE_COUNT).with_label_values(&[q]).set(0); + } + } + } + + if otel_enabled { + for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { + if queue_stats.get(q).is_none() { + otel_set_queue_count(q, 0); + } + } + } + + #[allow(unused_mut)] + let mut tags_to_watch = vec![]; + #[allow(unused_mut)] + let mut otel_tags_to_watch = vec![]; + for (tag, stat) in queue_stats.iter() { + let count = stat.count; + + #[cfg(feature = "prometheus")] + if metrics_enabled { + let metric = (*QUEUE_COUNT).with_label_values(&[tag]); + metric.set(count as i64); + tags_to_watch.push(tag.to_string()); + } + + if otel_enabled { + otel_tags_to_watch.push(tag.to_string()); + } + otel_set_queue_count(tag, count as i64); + } + + if save_metrics { + LAST_QUEUE_SAMPLE_MS.store(now_ms, Ordering::Relaxed); + save_queue_metrics(db, &queue_stats).await; + } + + if metrics_enabled { + let mut w = QUEUE_COUNT_TAGS.write().await; + *w = tags_to_watch; + } + if otel_enabled { + let mut w = OTEL_QUEUE_COUNT_TAGS.write().await; + *w = otel_tags_to_watch; + } } pub async fn reload_smtp_config(db: &Pool) { @@ -4905,6 +5657,7 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .dedicated_workers .as_ref() .is_some_and(|dws| !dws.is_empty()); + if **wc != config || has_dedicated { if kill_if_change { if has_dedicated @@ -4952,6 +5705,37 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b store_pull_query(&config).await; WORKER_CONFIG.store(std::sync::Arc::new(config)); } + + // After the store, so a retry that wakes mid-build reads the config being applied + // rather than the one it replaced. Unconditional rather than gated on the value + // changing, so that a pass triggered by anything else — a license-plan change, most + // of all — still re-evaluates the entitlement. + #[cfg(feature = "parquet")] + reload_cache_object_store_override_with_retry(db).await; + } +} + +/// Apply this worker group's dependency-cache object store, retrying once shortly after a build +/// that failed for a reason that may pass — the periodic settings reload behind it is 12h apart, +/// which is a long time for a whole group to cache nothing but locally. +#[cfg(feature = "parquet")] +pub async fn reload_cache_object_store_override_with_retry(db: &DB) { + let settings = WORKER_CONFIG.load().object_store_cache_config.clone(); + if matches!( + windmill_object_store::reload_cache_object_store_override(db, settings).await, + ObjectStoreReload::Later + ) { + let db = db.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + if windmill_object_store::cache_object_store_override_failed().await { + // Re-read rather than reuse: the group config may have changed while we slept, + // and installing the settings this retry was born with would pin the worker to a + // store the group no longer asks for. + let settings = WORKER_CONFIG.load().object_store_cache_config.clone(); + windmill_object_store::reload_cache_object_store_override(&db, settings).await; + } + }); } } @@ -5430,7 +6214,10 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n /// Force-complete a zombie job that handle_job_error failed to complete. /// This is a minimal fallback: it inserts a failed completed job and deletes /// from the queue in a single transaction, without schedule pushing or -/// error handler logic that could cause the completion to fail. +/// error handler logic. The one thing it keeps is the WAC parent notification, +/// deliberately inside the transaction: if that fails, the whole completion +/// rolls back and the job waits for the next sweep, which is cheaper than a +/// parent parked for its full suspend window and a task run twice. async fn force_complete_zombie_job( db: &Pool, job_id: &Uuid, @@ -5452,14 +6239,18 @@ async fn force_complete_zombie_job( "Zombie job {job_id} was not completed by handle_job_error, force-completing it" ); + // Same `{"error": ...}` shape as every other failed job's result, so a WAC + // parent's failure record reads the name and message like any task failure. let error_value = serde_json::json!({ - "message": error_message, - "name": "ExecutionErr", + "error": { + "message": error_message, + "name": "ExecutionErr", + } }); let mut tx = db.begin().await?; - sqlx::query!( + let duration_ms = sqlx::query_scalar!( "INSERT INTO v2_job_completed (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker) SELECT q.workspace_id, q.id, q.started_at, @@ -5468,19 +6259,50 @@ async fn force_complete_zombie_job( FROM v2_job_queue q LEFT JOIN v2_job_runtime r ON r.id = q.id WHERE q.id = $1 - ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb + RETURNING duration_ms AS \"duration_ms!\"", job_id, error_value, ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + // A WAC parent parked on this job must learn of the failure here too, or it + // waits out its whole suspend window and runs the task again. + let mut wac_parent_ready = false; + if let Some(duration_ms) = duration_ms { + let parent = sqlx::query!( + "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(&mut *tx) + .await?; + if let Some(parent_job) = parent + .filter(|j| j.flow_step_id.is_none()) + .and_then(|j| j.parent_job) + { + wac_parent_ready = windmill_common::wac::record_child_completion( + &mut tx, + &parent_job, + job_id, + false, + duration_ms, + &error_value.to_string(), + ) + .await?; + } + } + sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job_id) .execute(&mut *tx) .await?; tx.commit().await?; + if wac_parent_ready { + windmill_common::wac::WAC_SUSPEND_READY.store(true, Ordering::Relaxed); + } + tracing::info!("Force-completed zombie job {job_id}"); Ok(()) } @@ -6302,6 +7124,34 @@ pub fn apply_app_workspaced_route_setting(app_workspaced_route: Option error::Result<()> { + let v = + load_value_from_global_settings(conn, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING).await?; + apply_http_route_default_allowed_origins_setting(v) +} + +pub fn apply_http_route_default_allowed_origins_setting( + value: Option, +) -> error::Result<()> { + // A bad value leaves whatever is already loaded in place rather than + // reverting to no restriction. On the boot path that is still the empty + // default, so what keeps a stored typo from widening CORS instance-wide is + // write-time validation, not this. + let origins = match parse_allowed_origins_setting(value.as_ref()) { + Ok(origins) => origins, + Err(err) => { + tracing::error!( + "Invalid {} setting, keeping the previous value: {err:#}", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ); + return Ok(()); + } + }; + + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.store(std::sync::Arc::new(origins)); + Ok(()) +} + pub async fn reload_http_route_workspaced_route_setting(conn: &DB) -> error::Result<()> { let v = load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; apply_http_route_workspaced_route_setting(conn, v).await @@ -6367,7 +7217,6 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( Ok(()) } - pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { let v = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; apply_jwt_secret_setting(db, v).await @@ -6844,3 +7693,94 @@ mod zombie_worker_memory_pct_tests { ); } } + +#[cfg(test)] +mod otel_traces_retention_tests { + use super::{delete_expired_otel_traces, DB}; + + async fn insert_span(db: &DB, id: u8, age_secs: i64) { + sqlx::query!( + "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano) + VALUES ($1, $2, 'GET /', 3, $3, $3)", + &[id; 16][..], + &[id; 8][..], + (chrono::Utc::now() - chrono::Duration::seconds(age_secs)) + .timestamp_nanos_opt() + .unwrap(), + ) + .execute(db) + .await + .unwrap(); + } + + /// The cutoff crosses two units: a retention configured in seconds against a column holding + /// nanoseconds. Getting that conversion wrong is silent in both directions — a window a + /// billion times too wide never deletes anything, one a billion times too narrow deletes + /// every span on the next tick — so pin it on either side of the boundary. + #[sqlx::test(migrations = "./migrations")] + async fn deletes_only_spans_past_the_window(db: DB) -> anyhow::Result<()> { + let day = 60 * 60 * 24; + insert_span(&db, 1, 60).await; + insert_span(&db, 2, 6 * day).await; + insert_span(&db, 3, 8 * day).await; + + assert_eq!(delete_expired_otel_traces(&db, 7 * day).await, 1); + + let kept = sqlx::query_scalar!("SELECT trace_id FROM otel_traces ORDER BY trace_id") + .fetch_all(&db) + .await?; + assert_eq!(kept, vec![vec![1u8; 16], vec![2u8; 16]]); + Ok(()) + } +} + +#[cfg(test)] +mod log_file_listing_tests { + use super::{rotated_log_files, sorted_log_files}; + + fn names(files: Vec<(chrono::NaiveDateTime, String)>) -> Vec { + files.into_iter().map(|(_, n)| n).collect() + } + + /// A directory read newest-entry-first is what tmpfs actually hands back. + #[test] + fn orders_by_minute_whatever_order_readdir_used() { + let newest_first = [ + "h.log.2026-08-29-06-49", + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-47", + ]; + assert_eq!( + names(sorted_log_files(newest_first.iter().map(|x| x.to_string()))), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-49", + ] + ); + assert_eq!( + names(rotated_log_files( + newest_first.iter().map(|x| x.to_string()) + )), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + ] + ); + } + + #[test] + fn drops_names_that_are_not_rotated_log_files() { + let files = sorted_log_files( + ["h.log", "not-a-log-file", "h.log.2026-08-29-06-46"] + .iter() + .map(|x| x.to_string()), + ); + assert_eq!(files.len(), 1); + assert_eq!(files[0].1, "h.log.2026-08-29-06-46"); + assert_eq!(files[0].0.to_string(), "2026-08-29 06:46:00"); + } +} diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 7a2996a9b1..fdb926a014 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,10 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) +ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) +ai_shared_artifact: id(uuid), workspace_id(char), artifact_id(char), email(char), created_by(char), name(char), kind(char), version(int), content(text), shared_at(ts) + FK: (workspace_id) -> workspace(id) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) @@ -52,7 +56,7 @@ app_version_lite: id(bigint), value(jsonb) FK: (id) -> app_version(id) asset: workspace_id(char), path(char), kind(asset_kind), usage_access_type(asset_access_type), usage_path(char), usage_kind(asset_usage_kind), created_at(ts), id(bigint), columns(jsonb) FK: (workspace_id) -> workspace(id) -audit: workspace_id(char), id(bigint), timestamp(ts), username(char), operation(char), action_kind(action_kind), resource(char), parameters(jsonb), email(char), span(char) +audit_partitioned: workspace_id(char), id(bigint), timestamp(ts), username(char), operation(char), action_kind(action_kind), resource(char), parameters(jsonb), email(char), span(char) autoscaling_event: id(int), worker_group(text), event_type(autoscaling_event_type), desired_workers(int), applied_at(ts), reason(text) capture: workspace_id(char), path(char), created_at(ts), created_by(char), main_args(jsonb), is_flow(bool), trigger_kind(trigger_kind), preprocessor_args(jsonb), id(bigint) FK: (workspace_id) -> workspace(id) @@ -68,11 +72,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) -dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char), id(bigint) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) @@ -82,7 +90,7 @@ debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(t debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) dependency_map: workspace_id(char), importer_path(char), importer_kind(importer_kind), imported_path(char), importer_node_id(char) -deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid) +deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid), id(bigint) FK: (workspace_id) -> workspace(id) draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at(ts) FK: (workspace_id) -> workspace(id) @@ -91,9 +99,9 @@ email_trigger: path(char), local_part(char), workspaced_local_part(bool), script favorite: usr(char), workspace_id(char), path(char), favorite_kind(favorite_kind) flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]) FK: (workspace_id) -> workspace(id) -flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char) +flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char), is_test(bool) FK: (workspace_id) -> workspace(id) -flow_conversation_message: id(uuid), conversation_id(uuid), message_type(message_type), content(text), job_id(uuid), created_at(ts), created_seq(int8), step_name(char), success(bool) +flow_conversation_message: id(uuid), conversation_id(uuid), message_type(message_type), content(text), job_id(uuid), created_at(ts), created_seq(int8), step_name(char), success(bool), tool_arguments(text), tool_result(text), reasoning(text), attachments(jsonb) FK: (conversation_id) -> flow_conversation(id) | (job_id) -> v2_job(id) flow_iterator_data: job_id(uuid), itered(jsonb) flow_node: id(bigint), workspace_id(char), hash(bigint), path(char), lock(text), code(text), flow(jsonb), hash_v2(char(64)) @@ -107,13 +115,19 @@ folder: name(char), workspace_id(char), display_name(char), owners(char), extra_ folder_permission_history: id(bigint), workspace_id(char), folder_name(char), changed_by(char), changed_at(ts), change_type(char), affected(char) FK: (workspace_id, folder_name) -> folder(workspace_id, name) gcp_trigger: gcp_resource_path(char), topic_id(char), subscription_id(char), delivery_type(delivery_mode), delivery_config(jsonb), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), subscription_mode(gcp_subscription_mode), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), auto_acknowledge_msg(bool), ack_deadline(int), mode(trigger_mode), labels(text[]) +git_sync_ci_test_check: workspace_id(char), head_sha(char), head_ref(char), poster_workspace_id(char), repo_url(text), repo_resource_path(char), check_run_id(bigint), created_at(timestamptz), concluded(bool), conclusion(text), concluded_at(timestamptz), github_posted(bool) + FK: (workspace_id) -> workspace(id) + FK: (poster_workspace_id) -> workspace(id) +git_sync_synced_head: workspace_id(char), repo_resource_path(char), branch(char), sha(char), source(char), job_id(uuid), synced_at(timestamptz), tests_dispatched_at(timestamptz), ci_test_job_ids(uuid[]) + FK: (workspace_id) -> workspace(id) global_settings: name(char), value(jsonb), updated_at(ts) +guest_activity: email(char), workspace_id(char), day(date), last_seen_at(timestamptz), jwt_entry(bool) group_: workspace_id(char), name(char), summary(text), extra_perms(jsonb) FK: (workspace_id) -> workspace(id) group_permission_history: id(bigint), workspace_id(char), group_name(char), changed_by(char), changed_at(ts), change_type(char), member_affected(char) FK: (workspace_id, group_name) -> group_(workspace_id, name) healthchecks: id(bigint), check_type(text), healthy(bool), created_at(ts) -http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) +http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), allowed_origins(text[]), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) input: id(uuid), workspace_id(char), runnable_id(char), runnable_type(runnable_type), name(text), args(jsonb), created_at(ts), created_by(char), is_public(bool) FK: (workspace_id) -> workspace(id) instance_group: name(char), summary(char), id(char), scim_display_name(char), external_id(char) @@ -128,7 +142,7 @@ job_stats: workspace_id(char), job_id(uuid), metric_id(char), metric_name(char), kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), topic(char), partition(int), offset(bigint), created_at(ts) FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path) kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[]) -log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool) +log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool), indexed_at(ts) macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts) FK: (workspace_id) -> workspace(id) macro_usage: workspace_id(char), consumer_path(char), macro_name(char) @@ -140,9 +154,9 @@ mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char mcp_oauth_server_client: client_id(char), client_name(char), redirect_uris(text[]), created_at(ts) mcp_oauth_server_code: code(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), redirect_uri(text), code_challenge(char), code_challenge_method(char), created_at(ts), expires_at(ts) FK: (client_id) -> mcp_oauth_server_client(client_id) -metrics: id(char), value(jsonb), created_at(ts) +metrics: id(char), value(jsonb), created_at(ts), row_id(bigint) mqtt_trigger: mqtt_resource_path(char), subscribe_topics(jsonb[]), client_version(mqtt_client_version), v5_config(jsonb), v3_config(jsonb), client_id(char), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) -native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts) +native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts), enabled(bool) FK: (workspace_id) -> workspace(id) nats_trigger: path(char), nats_resource_path(char), subjects(char), stream_name(char), consumer_name(char), use_jetstream(bool), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) FK: (workspace_id) -> workspace(id) @@ -159,7 +173,7 @@ raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at( FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char), labels(text[]) FK: (workspace_id) -> workspace(id) -resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool) +resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool), display_name(char) FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) @@ -218,9 +232,9 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint), runnable_is_agent(bool) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), variable_expiration_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int), add_admins_and_developers_to_forks(bool), variable_expiration_handler(jsonb) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index 791556b035..4d6fffa346 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -293,6 +293,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> let lfs_config = json!({ "type": "FilesystemStorage", "root_path": storage_root, + "volume_storage": "primary", "public_resource": null, "advanced_permissions": null }); @@ -306,7 +307,11 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> .await?; // 2. Pre-populate the volume with a file - let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); std::fs::create_dir_all(&vol_dir)?; std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; @@ -343,6 +348,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> // 4. GET /file/* — download the existing file let resp = http .get(format!("{vol_base}/file/hello.txt")) + .query(&[("worker_name", "test-worker-1")]) .send() .await?; assert!( @@ -360,6 +366,7 @@ async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> // 5. PUT /file/* — upload a new file let resp = http .put(format!("{vol_base}/file/output.txt")) + .query(&[("worker_name", "test-worker-1")]) .body(b"written by agent worker".to_vec()) .send() .await?; @@ -432,7 +439,8 @@ async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow: "type": "FilesystemStorage", "root_path": storage_root, "public_resource": null, - "advanced_permissions": null + "advanced_permissions": null, + "volume_storage": "primary" }); sqlx::query!( @@ -444,20 +452,24 @@ async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow: .await?; // 2. Pre-populate the volume with a file - let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); std::fs::create_dir_all(&vol_dir)?; std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; // 3. Push the job, then run worker with HTTP connection (bun tag) - let code = r#"// volume: test-vol /tmp/data + let code = r#"// volume: test-vol data import { readFileSync, writeFileSync, existsSync } from "fs"; export function main() { - const content = readFileSync("/tmp/data/hello.txt", "utf-8"); - writeFileSync("/tmp/data/output.txt", "written by agent worker"); + const content = readFileSync("data/hello.txt", "utf-8"); + writeFileSync("data/output.txt", "written by agent worker"); return { read_content: content, - output_exists: existsSync("/tmp/data/output.txt"), + output_exists: existsSync("data/output.txt"), }; }"#; @@ -528,6 +540,7 @@ async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result< let lfs_config = json!({ "type": "FilesystemStorage", "root_path": storage_root, + "volume_storage": "primary", "public_resource": null, "advanced_permissions": null }); diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs new file mode 100644 index 0000000000..6b33a4141e --- /dev/null +++ b/backend/tests/ai_sessions.rs @@ -0,0 +1,1582 @@ +//! The AI session backup routes (`/w/{w}/ai/sessions/*`): a browser pushes pieces of its +//! sessions into the workspace's object storage and pulls them back whole. Pinned against a +//! FilesystemStorage LFS so the test needs no object store, which also lets it read what +//! landed on disk: the objects must be ciphertext, since bucket credentials are shared far +//! more widely than a user's transcripts. +#![cfg(all(feature = "private", feature = "parquet"))] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_common::utils::calculate_hash; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +async fn configure_primary_lfs(db: &Pool, root_path: &str) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + json!({ + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": null, + "advanced_permissions": null + }), + "test-workspace" + ) + .execute(db) + .await?; + Ok(()) +} + +/// Configures the primary storage through the route, which is what sweeps the workspace's +/// backups out of the instance store. +async fn configure_primary_lfs_via_route(base: &str, root_path: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": { + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": false, + "advanced_permissions": null, + "secondary_storage": {} + }})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The instance setting allowing the instance store to stand in for a workspace without +/// storage: `None` leaves it unset, which is on. +async fn set_instance_fallback(db: &Pool, on: Option) -> anyhow::Result<()> { + sqlx::query("DELETE FROM global_settings WHERE name = 'ai_sessions_instance_storage_fallback'") + .execute(db) + .await?; + if let Some(on) = on { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('ai_sessions_instance_storage_fallback', $1)", + ) + .bind(json!(on)) + .execute(db) + .await?; + } + Ok(()) +} + +/// Polls until nothing is under the directory, for a deletion that runs off the request. +async fn wait_until_empty(dir: &std::path::Path, what: &str) { + for _ in 0..100 { + if files_under(dir).is_empty() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("{what}: objects left under {}", dir.display()); +} + +async fn list(base: &str, token: &str) -> anyhow::Result { + let resp = authed(client().get(format!("{base}/ai/sessions/list")), token) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn pull(base: &str, token: &str, ids: &[&str]) -> anyhow::Result { + let resp = authed(client().post(format!("{base}/ai/sessions/pull")), token) + .json(&json!({ "ids": ids })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn push(base: &str, token: &str, body: Value) -> anyhow::Result { + Ok( + authed(client().post(format!("{base}/ai/sessions/push")), token) + .json(&body) + .send() + .await?, + ) +} + +fn copy_dir(from: &std::path::Path, to: &std::path::Path) -> std::io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.path().is_dir() { + copy_dir(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), target)?; + } + } + Ok(()) +} + +async fn rotate(base: &str, key: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/encryption_key")), + "SECRET_TOKEN", + ) + .json(&json!({ "new_key": key })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The user's prefix on disk, `windmill_ai_sessions/{w_id}/g{generation}/{email hash}`, +/// under the newest generation: deleting an older generation's objects leaves its +/// directories behind, and `read_dir` order differs across filesystems. +fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { + let workspace = storage_dir.join("windmill_ai_sessions/test-workspace"); + let hash = calculate_hash(email); + std::fs::read_dir(&workspace) + .ok() + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let generation: i64 = entry + .file_name() + .to_str()? + .strip_prefix('g')? + .parse() + .ok()?; + Some((generation, entry.path().join(&hash))) + }) + .filter(|(_, path)| path.exists()) + .max_by_key(|(generation, _)| *generation) + .map(|(_, path)| path) + .expect("the user has backups under the current key") +} + +/// Every file under the storage root, as bytes. +fn files_under(root: &std::path::Path) -> Vec<(std::path::PathBuf, Vec)> { + let mut out = vec![]; + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push((path.clone(), std::fs::read(&path).unwrap_or_default())); + } + } + } + out +} + +#[sqlx::test(fixtures("base"))] +async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // No storage configured, and the instance store (another test of this process may + // have loaded one) not allowed to stand in: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], false); + assert_eq!(listing["sessions"], json!([])); + + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let chat = json!({ "id": "c1", "sessionId": "s1", "title": "MARKER_PLAINTEXT_TITLE", "lastModified": 2, + "actualMessages": [], "displayMessages": [{"role": "user", "content": "hello"}] }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ + "id": "s1", + "whole": true, + "head": head, + "chats": [{ "id": "c1", "record": chat }, { "id": "c2", "record": { "id": "c2" } }], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // A part more parts follow names its push, or the session would stay listed between + // the parts: one that does not is refused before anything of it lands. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "partial": true, "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s1" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .len(), + 2 + ); + + // A part with more of the session to follow lists nothing; the part that completes + // the push does, newest first. + let ids = |listing: &Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "opens": true, "head": { "id": "s2", "workspace_id": "test-workspace", "createdAt": 2, "chatId": "c" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s2", "s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s2"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A session that outgrew one answer (three chats of 12 MB against the 32 MB budget) + // comes in pages, each naming where the next picks up, and nothing is left out. + let big = "y".repeat(12 * 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "opens": true, "head": { "id": "s4", "workspace_id": "test-workspace", "createdAt": 4, "chatId": "c1" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for cid in ["c1", "c2", "c3"] { + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "chats": [{ "id": cid, "record": { "id": cid, "big": big } }], "partial": cid != "c3" }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut pages = vec![]; + let mut resume = json!(null); + loop { + let body = if resume.is_null() { + json!({ "ids": ["s4"] }) + } else { + json!({ "ids": ["s4"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = pulled["sessions"][0].clone(); + assert_eq!(page["id"], "s4"); + resume = page["next"].clone(); + pages.push(page); + if resume.is_null() { + break; + } + assert!(pages.len() < 5, "a paged pull must end"); + } + assert!(pages.len() >= 2, "36 MB must not fit one answer"); + // Every page of an unchanged backup carries the same listing fingerprint; a chat added + // to the session changes it, which is what tells a browser its pages do not belong + // together any more. + let listing = pages[0]["listing"].clone(); + assert!(listing.is_string()); + assert!(pages.iter().all(|p| p["listing"] == listing)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 1 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + // So does a chat rewritten at the same size: the fingerprint takes in the entity tag, + // not only the size and a modification time the store may report coarsely. + let listing = pulled["sessions"][0]["listing"].clone(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 2 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + let mut chat_ids: Vec = pages + .iter() + .flat_map(|p| p["chats"].as_array().unwrap().iter()) + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect(); + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s4"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // More objects than a page keeps listing metadata for, from a store that lists in no + // order: the pages still carry every one of them, each once. + let many = 5001; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "opens": true, "head": { "id": "s5", "workspace_id": "test-workspace", "createdAt": 5, "chatId": "c00000" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for start in (0..many).step_by(100) { + let chats: Vec = (start..(start + 100).min(many)) + .map(|i| json!({ "id": format!("c{i:05}"), "record": { "id": format!("c{i:05}") } })) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "chats": chats, "partial": start + 100 < many }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut seen = std::collections::HashSet::new(); + let mut resume = json!(null); + let mut pages = 0; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s5"] }) + } else { + json!({ "ids": ["s5"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = &pulled["sessions"][0]; + for c in page["chats"].as_array().unwrap() { + assert!( + seen.insert(c["id"].as_str().unwrap().to_string()), + "a chat came twice" + ); + } + pages += 1; + resume = page["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a paged pull must end"); + } + assert_eq!(pages, 2); + assert_eq!(seen.len(), many); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s5"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A head at exactly its cap round-trips: the ciphertext read back is a block larger. + let mut big_head = json!({ "id": "s3", "workspace_id": "test-workspace", "createdAt": 3, "chatId": "c", "pad": "" }); + let pad = 1024 * 1024 - serde_json::to_string(&big_head)?.len(); + big_head["pad"] = json!("x".repeat(pad)); + assert_eq!(serde_json::to_string(&big_head)?.len(), 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s3", "whole": true, "head": big_head }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s3"]).await?; + assert_eq!(pulled["sessions"][0]["head"], big_head); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s3"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // An object larger than any push writes, planted with the bucket's credentials at a + // predictable key, is not read. + let planted = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/planted/head.json"); + std::fs::create_dir_all(planted.parent().unwrap())?; + std::fs::File::create(&planted)?.set_len(32 * 1024 * 1024 + 1)?; + let pulled = pull(&base, "SECRET_TOKEN", &["planted"]).await?; + assert_eq!(pulled["sessions"], json!([])); + std::fs::remove_dir_all(planted.parent().unwrap())?; + // Under a session that exists, a planted chat is skipped without buffering and without + // the page ending before it, so the pull neither balloons nor loops. + let planted_chat = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s1/chats/planted.json"); + std::fs::File::create(&planted_chat)?.set_len(32 * 1024 * 1024 + 1)?; + let mut resume = json!(null); + let mut pages = 0; + let mut chat_ids = vec![]; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s1"] }) + } else { + json!({ "ids": ["s1"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + assert_eq!(pulled["sessions"][0]["head"], head); + for c in pulled["sessions"][0]["chats"].as_array().unwrap() { + chat_ids.push(c["id"].as_str().unwrap().to_string()); + } + pages += 1; + resume = pulled["sessions"][0]["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a planted object must not keep the pull going"); + } + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2"]); + std::fs::remove_file(&planted_chat)?; + + let pulled = pull(&base, "SECRET_TOKEN", &["s1", "never-pushed"]).await?; + assert_eq!(pulled["deferred"], json!([])); + let sessions = pulled["sessions"].as_array().unwrap(); + assert_eq!(sessions.len(), 1, "an id with no backup is simply absent"); + let s1 = &sessions[0]; + assert_eq!(s1["head"], head); + let mut chats = s1["chats"].as_array().unwrap().clone(); + chats.sort_by_key(|c| c["id"].as_str().unwrap().to_string()); + assert_eq!(chats[0]["record"], chat); + assert_eq!(chats[1]["id"], "c2"); + assert_eq!( + s1["images"], + json!([{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }]) + ); + assert_eq!(s1["artifacts"], json!({ "items": [], "versions": [] })); + + // Nothing on disk carries the transcript in the clear. + let files = files_under(storage_dir.path()); + assert!( + files.len() >= 4, + "expected the pushed objects on disk, got {files:?}" + ); + for (path, bytes) in &files { + let text = String::from_utf8_lossy(bytes); + assert!( + !text.contains("MARKER_PLAINTEXT_TITLE") && !text.contains("base64,AAAA"), + "{} holds plaintext", + path.display() + ); + } + let key_paths: Vec = files + .iter() + .map(|(p, _)| { + p.strip_prefix(storage_dir.path()) + .unwrap() + .to_string_lossy() + .to_string() + }) + .collect(); + assert!( + key_paths + .iter() + .all(|p| p.starts_with("windmill_ai_sessions/test-workspace/") + && !p.contains("test@windmill.dev")), + "keys carry the workspace and never the email: {key_paths:?}" + ); + + // Another member of the workspace sees none of it. + let other = list(&base, "SECRET_TOKEN_2").await?; + assert_eq!(other["enabled"], true); + assert_eq!(other["sessions"], json!([])); + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!(other["sessions"], json!([])); + + // Nor after copying the first user's ciphertext under their own prefix, which anyone + // holding the bucket credentials can do: the key is bound to the user, not the workspace. + let first = user_root(storage_dir.path(), "test@windmill.dev"); + let second = first + .parent() + .unwrap() + .join(calculate_hash("test2@windmill.dev")); + copy_dir(&first, &second)?; + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!( + other["sessions"], + json!([]), + "relocated ciphertext must not decrypt for another user" + ); + std::fs::remove_dir_all(&second)?; + + // Deleting a chat takes its images along; a head-only push leaves the rest in place. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + let s1 = &pulled["sessions"][0]; + assert_eq!(s1["head"], head); + assert_eq!(s1["chats"].as_array().unwrap().len(), 1); + assert_eq!(s1["chats"][0]["id"], "c2"); + assert_eq!(s1["images"], json!([])); + + // Rotating the workspace key moves the routes to a fresh generation's prefix and deletes + // the older ones off the request rather than re-key anything; the answers name the new + // generation (`backup_generation`), which is what makes every browser push its sessions + // whole again, while `storage_id` names the storage and stays. + let before = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + for _ in 0..100 { + if files_under(storage_dir.path()).is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + files_under(storage_dir.path()).is_empty(), + "a rotation must leave no backup object behind" + ); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert_eq!(listing["storage_id"], before["storage_id"]); + assert_ne!( + listing["backup_generation"], before["backup_generation"], + "a rotation must bump the backup generation" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"], + json!([]) + ); + // The browser's next push fills the storage back under the new key. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["sessions"][0]["head"], head); + assert_eq!(pulled["sessions"][0]["chats"][0]["id"], "c2"); + // Setting the key already in place is not a rotation the browsers would notice, so it + // keeps the backups. + let same = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["storage_id"], same["storage_id"]); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["head"], + head + ); + + // A push that does not open the session whole rides on the head in the storage; with + // none there (another device removed the backup, or nothing was ever pushed) it is + // refused and lists nothing, head or no head on it, until the session goes whole. + let s6_head = + json!({ "id": "s6", "workspace_id": "test-workspace", "createdAt": 6, "chatId": "c" }); + let not_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s6") + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "whole": true, "head": s6_head, "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!(answer["results"][0]["needs_whole"].is_null()); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s6" + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s6"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "head": s6_head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + assert!( + files_under(&user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s6")) + .is_empty() + ); + + // Between the parts of a whole push (head landed, marker not yet), an incremental push + // from another device is refused too: it rides on a listed session, and there is none + // until the last part, which lists it. + let s8_head = + json!({ "id": "s8", "workspace_id": "test-workspace", "createdAt": 8, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "opens": true, "head": s8_head, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "chats": [{ "id": "c9", "record": { "id": "c9" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s8") + }; + assert!(!listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "chats": [{ "id": "c1", "record": { "id": "c1" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s8"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect::>(), + vec!["c1"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s8"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A whole push replaces the backup: what the storage held of the session and the new + // push does not carry (a chat deleted while the workspace was on another storage) goes. + let s9_head = + json!({ "id": "s9", "workspace_id": "test-workspace", "createdAt": 9, "chatId": "c1" }); + let s9_chats = |ids: &[&str]| -> Vec { + ids.iter() + .map(|c| json!({ "id": c, "record": { "id": c } })) + .collect() + }; + let pulled_chats = |pulled: Value| -> Vec { + pulled["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "head": s9_head, "chats": s9_chats(&["c1", "c2"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c2"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "epoch": 1, "head": s9_head, "chats": s9_chats(&["c1"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1"] + ); + // The marker carries the move count the push named, once; an incremental push at + // another count rides on nothing, one at the same count lands. + let s9_epochs = |listing: Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["id"] == "s9") + .map(|s| s["epoch"].clone()) + .collect() + }; + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7"] + ); + + // An incremental push split over parts unlists the session while it is in progress (a + // pull between two parts would take a mix of old and new pieces for the backup) and + // lists it again with the last part; while one is in progress or abandoned, a push that + // is not part of it is refused, so the browser's next push of the session goes whole. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "opens": true, "chats": s9_chats(&["c8"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + Vec::::new() + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c11"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "chats": s9_chats(&["c9"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7", "c8", "c9"] + ); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + + // Two devices pushing the session whole at once: the push that opened later replaced + // the earlier one's pieces, so the earlier one's last part is refused and lists nothing, + // and the session is listed with the later push's pieces alone. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "opens": true, "head": s9_head, "chats": s9_chats(&["c3"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "opens": true, "head": s9_head, "chats": s9_chats(&["c4"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "chats": s9_chats(&["c5"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s9")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "chats": s9_chats(&["c6"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c4", "c6"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s9"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // An incremental part changing more than one object unlists the session before its + // writes, so one write failing after another landed leaves it absent rather than listed + // as a mix of old and new pieces. A directory planted at `artifacts.json` fails that write. + let s10_head = + json!({ "id": "s10", "workspace_id": "test-workspace", "createdAt": 10, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1"]), "artifacts": { "items": ["a1"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let s10_dir = user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s10"); + let artifacts_path = s10_dir.join("artifacts.json"); + std::fs::remove_file(&artifacts_path)?; + std::fs::create_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c2"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!( + answer["results"][0]["error"].is_string(), + "the artifacts write must fail: {answer}" + ); + assert!(answer["results"][0]["needs_whole"].is_null()); + assert!( + s10_dir.join("chats/c2.json").is_file(), + "the chat landed before the artifacts failed" + ); + let s10_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s10") + }; + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s10"]).await?["sessions"], + json!([]) + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c3"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + std::fs::remove_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1", "c2", "c3"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(s10_listed(list(&base, "SECRET_TOKEN").await?)); + let pulled = pull(&base, "SECRET_TOKEN", &["s10"]).await?; + assert_eq!( + pulled["sessions"][0]["artifacts"], + json!({ "items": ["a2"] }) + ); + assert_eq!(pulled_chats(pulled), vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s10"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // Removal empties both prefixes. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s1"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert!( + listing["storage_id"].is_string(), + "an answer names its storage: {listing}" + ); + assert!( + files_under(storage_dir.path()).is_empty(), + "removal must leave no object behind" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base", "jobs_read_auth"))] +async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + // A push prepared for another user must not land under the caller's prefix. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test2@windmill.dev", "sessions": [{ "id": "s1", "head": { "id": "s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 409, "{}", resp.text().await?); + + // Ids are what the server builds keys from. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "../s1", "head": { "id": "../s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A whole push opens with its head; one without is refused before anything of it lands, + // and nothing lists the session. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s7", "whole": true, "chats": [{ "id": "c", "record": { "id": "c" } }] }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s7")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["a/b"] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // An image is a base64 data URL, stored and served verbatim; anything JSON would have + // to escape (and so inflate past the pull budget) is refused. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "images": [{ "chat_id": "c1", "id": "i1", "data_url": "data:image/png;base64,\u{0001}\u{0001}\"" }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // Nested lists are bounded too: each entry is an object-store call. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": (0..1001).map(|i| format!("c{i}")).collect::>() }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // ...and across the whole request, not only per entry. + let sessions: Vec = (0..100) + .map(|i| { + json!({ "id": format!("s{i}"), "delete_chats": (0..50).map(|j| format!("c{j}")).collect::>() }) + }) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": sessions }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A pull body is a handful of ids; a large one is refused before it is parsed. + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .header("Content-Type", "application/json") + .body(format!("{{\"ids\":[\"{}\"]}}", "a".repeat(100_000))) + .send() + .await?; + assert_eq!(resp.status(), 413, "{}", resp.text().await?); + + // A scoped token (here `jobs:read`) is minted for something narrower than the user's + // whole assistant history. + let resp = authed( + client().get(format!("{base}/ai/sessions/list")), + "SCOPED_DENO_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + + assert!(files_under(storage_dir.path()).is_empty()); + Ok(()) +} + +/// Sets the object's modification time `days` back: the FilesystemStorage answers +/// `last_modified` from it, so this is a session no push touched since. +fn age_object(path: &std::path::Path, days: u64) -> std::io::Result<()> { + let at = std::time::SystemTime::now() - std::time::Duration::from_secs(days * 86_400); + std::fs::File::options() + .write(true) + .open(path)? + .set_modified(at) +} + +#[sqlx::test(fixtures("base"))] +async fn test_expired_backups_are_swept_by_age_and_left_out_of_the_listing( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let chat = |sid: &str, cid: &str| { + json!({ "id": cid, "record": { "id": cid, "sessionId": sid, "lastModified": 2, + "actualMessages": [], "displayMessages": [] } }) + }; + let whole = |sid: &str| { + json!({ + "id": sid, "whole": true, "epoch": 0, + "head": { "id": sid, "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }, + "chats": [chat(sid, "c1")], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }) + }; + // Two pushes split over parts of which only the first part landed: one a browser + // abandoned long ago (its token aged past the retention), one still in flight. + let opening = |sid: &str| { + json!({ + "id": sid, "whole": true, "epoch": 0, "push": format!("t-{sid}"), "opens": true, + "partial": true, "chats": [chat(sid, "c1")], + "head": { "id": sid, "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" } + }) + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [ + whole("old"), whole("live"), opening("abandoned"), opening("inflight") + ] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let root = user_root(storage_dir.path(), "test@windmill.dev"); + age_object(&root.join("index/old/0"), 40)?; + age_object(&root.join("index/abandoned/push"), 40)?; + + let listed = |listing: Value| -> Vec { + let mut ids: Vec = listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect(); + ids.sort(); + ids + }; + let objects = |root: &std::path::Path| -> Vec { + files_under(root) + .into_iter() + .map(|(p, _)| p.strip_prefix(root).unwrap().to_string_lossy().into_owned()) + .collect() + }; + + // Without a retention nothing is swept, however old. + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live", "old"]); + + let set_retention = |days: Value| { + authed( + client().post(format!("{base}/workspaces/edit_copilot_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "sessions_retention_days": days })) + .send() + }; + let resp = set_retention(json!(0)).await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + let resp = set_retention(json!(30)).await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // The listing leaves the expired session out before the sweep reaches it. + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + assert!(root.join("sessions/old/head.json").exists()); + + // A removal cut short (a directory stands where the head is, so it cannot be unlinked) + // leaves the sweep's record with the markers gone; the next pass finds it and finishes. + let head = root.join("sessions/old/head.json"); + std::fs::remove_file(&head)?; + std::fs::create_dir(&head)?; + std::fs::write(head.join("planted"), b"")?; + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert!(root.join("index/old/sweep").exists()); + assert!(!root.join("index/old/0").exists()); + assert!(root.join("sessions/old/chats/c1.json").exists()); + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + std::fs::remove_dir_all(&head)?; + + windmill_api::sweep_expired_ai_session_backups(&db).await; + let remaining = objects(&root); + assert!( + remaining + .iter() + .all(|p| !p.contains("/old/") && !p.contains("/abandoned/")), + "{remaining:?}" + ); + for kept in [ + "index/live/0", + "sessions/live/head.json", + "sessions/live/chats/c1.json", + "images/live/c1/img1", + "index/inflight/push", + "sessions/inflight/head.json", + ] { + assert!( + remaining.iter().any(|p| p == kept), + "{kept} in {remaining:?}" + ); + } + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + + // A second pass has nothing to do; a session pushed again since its marker aged is + // renewed by the push, which rewrites the marker. + windmill_api::sweep_expired_ai_session_backups(&db).await; + age_object(&root.join("index/live/0"), 40)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", + "sessions": [{ "id": "live", "epoch": 0, "chats": [chat("live", "c2")] }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + windmill_api::sweep_expired_ai_session_backups(&db).await; + let mut after = objects(&root); + after.sort(); + let mut expected = remaining.clone(); + expected.push("sessions/live/chats/c2.json".to_string()); + expected.sort(); + assert_eq!(after, expected); + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + Ok(()) +} + +/// Puts the process-wide instance store back to none, even when an assertion fails. +struct ResetInstanceStore; +impl Drop for ResetInstanceStore { + fn drop(&mut self) { + if let Ok(mut store) = windmill_object_store::OBJECT_STORE_SETTINGS.try_write() { + *store = None; + } + } +} + +/// Puts the process-wide license key id back to none, an Enterprise plan in this build, +/// even when an assertion fails. +struct ResetLicensePlan; +impl Drop for ResetLicensePlan { + fn drop(&mut self) { + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new(String::new())); + } +} + +/// A workspace without storage of its own backs up to the instance object store, every +/// answer saying so (`fallback`); a storage of its own, once configured, answers instead, +/// under a generation past everything the workspace left in the instance store, which the +/// change deletes; a plan switched to Pro stops the fallback with the store still loaded. +#[sqlx::test(fixtures("base"))] +async fn test_backups_fall_back_to_the_instance_storage(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // The instance store, built from its settings as `reload_object_store_setting` does. + let instance_dir = tempfile::tempdir()?; + let instance_root = instance_dir.path().to_string_lossy().to_string(); + *windmill_object_store::OBJECT_STORE_SETTINGS.write().await = Some( + windmill_object_store::build_object_store_from_settings( + windmill_object_store::ObjectSettings::Filesystem( + windmill_object_store::FilesystemSettings { root_path: instance_root.clone() }, + ), + None, + ) + .await?, + ); + let _reset = ResetInstanceStore; + let in_instance = instance_dir + .path() + .join("windmill_ai_sessions/test-workspace"); + + // Turned off by the instance setting: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + set_instance_fallback(&db, None).await?; + + // On, as it is unless turned off: the backups land in the instance store, under the + // workspace's prefix, and every answer says which kind of store it came from. + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let entry = json!({ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c1", "record": { "id": "c1" } }] }); + let push_whole = || { + push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [entry.clone()] }), + ) + }; + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["fallback"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["fallback"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + let fallback_storage_id = listing["storage_id"].clone(); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["fallback"], true); + assert_eq!(pulled["sessions"][0]["head"], head); + assert!(!files_under(&in_instance).is_empty()); + + // The workspace's storage usage counts them, under a name of their own. + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + let fallback_usage = usage["storages"] + .as_array() + .unwrap() + .iter() + .find(|s| s["storage"] == "_ai_sessions_fallback_") + .unwrap_or_else(|| panic!("no fallback usage in {usage}")); + assert!(fallback_usage["bytes"].as_i64().unwrap() > 0); + + // The retention sweep reaches what the instance store keeps for the workspace, choosing + // that store from the row it reads the generation from. + let instance_user = user_root(instance_dir.path(), "test@windmill.dev"); + age_object(&instance_user.join("index/s1/0"), 40)?; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 30}' \ + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert!(!instance_user.join("index/s1/0").exists()); + assert!(!instance_user.join("sessions/s1/head.json").exists()); + // Pushed again, so the rotation below has a backup to delete. + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + + // A key rotation sweeps the older generation out of the instance store too. + rotate(&base, &"c".repeat(64)).await?; + wait_until_empty(&in_instance, "a rotation on the instance store").await; + assert_eq!(list(&base, "SECRET_TOKEN").await?["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + + // A storage of its own answers instead, under a generation the configuration moved past + // everything the workspace left in the instance store: nothing there is read again, + // whichever store a later return to the fallback finds, and it is deleted. + let before = list(&base, "SECRET_TOKEN").await?; + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs_via_route(&base, &storage_dir.path().to_string_lossy()).await?; + wait_until_empty(&in_instance, "configuring a workspace storage").await; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_ne!(listing["storage_id"], fallback_storage_id); + assert_eq!( + listing["backup_generation"].as_i64(), + before["backup_generation"].as_i64().map(|g| g + 1), + "configuring a storage over the fallback must move the generation on" + ); + assert_eq!(listing["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(storage_dir.path()).is_empty()); + assert!(files_under(&in_instance).is_empty()); + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage?refresh=true")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + assert!( + !usage.to_string().contains("_ai_sessions_fallback_"), + "nothing is counted in the instance store for a workspace with storage: {usage}" + ); + + // Pointed at the instance store's own bucket, a storage of its own keeps its live + // backups there under the current generation, which no storage change deletes. + configure_primary_lfs_via_route(&base, &instance_root).await?; + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + let same = list(&base, "SECRET_TOKEN").await?; + configure_primary_lfs_via_route(&base, &instance_root).await?; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + assert!(!files_under(&in_instance).is_empty()); + let listing = list(&base, "SECRET_TOKEN").await?; + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // Back to no storage of its own, the fallback answers; a plan switched to Pro while the + // instance store stays loaded stops it at once, for the listing and the push alike. + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!(list(&base, "SECRET_TOKEN").await?["fallback"], true); + let _enterprise_again = ResetLicensePlan; + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new("test_pro".to_string())); + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], false, "{pushed}"); + + Ok(()) +} diff --git a/backend/tests/ai_shared_artifacts.rs b/backend/tests/ai_shared_artifacts.rs new file mode 100644 index 0000000000..17dd313755 --- /dev/null +++ b/backend/tests/ai_shared_artifacts.rs @@ -0,0 +1,184 @@ +//! Shared AI session artifacts: one link per author and artifact, readable by any workspace +//! member until its retention window passes, and removable only by its author or an admin. +//! +//! Expiry is enforced on read as well as by the monitor's sweep, so a share past its window must +//! not be served in the gap before the sweep reaches it. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN: &str = "Bearer SECRET_TOKEN"; +const MEMBER: &str = "Bearer SECRET_TOKEN_2"; + +async fn share( + client: &reqwest::Client, + base: &str, + token: &str, + content: &str, +) -> anyhow::Result { + let resp = client + .post(format!("{base}/share")) + .header("Authorization", token) + .json(&json!({ + "artifact_id": "plan:session-1", + "name": "Plan", + "kind": "md", + "version": 1, + "content": content, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +#[sqlx::test(fixtures("base", "ai_shared_artifacts"))] +async fn shared_artifact_is_served_to_members_until_it_expires( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let first = share(&client, &base, MEMBER, "draft").await?; + let second = share(&client, &base, MEMBER, "final").await?; + assert_eq!(first["id"], second["id"], "re-sharing minted a second link"); + let id = second["id"].as_str().unwrap(); + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["content"], "final"); + + // The handlers read through the raw pool, so the workspace in the URL is the only thing + // scoping a share: a member of another workspace must not reach it by id through theirs. + let resp = client + .get(format!( + "http://localhost:{}/api/w/test-workspace-2/ai/shared_artifacts/get/{id}", + server.addr.port() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a share was served through another workspace's path" + ); + + sqlx::query( + "UPDATE ai_shared_artifact SET shared_at = now() - ($1::bigint + 60) * interval '1 second'", + ) + .bind(windmill_common::ai_shared_artifact_retention_secs()) + .execute(&db) + .await?; + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 404, "an expired share was served"); + + let status: Value = client + .get(format!("{base}/status?artifact_id=plan:session-1")) + .header("Authorization", MEMBER) + .send() + .await? + .json() + .await?; + assert!( + status.get("share").is_none(), + "an expired share was reported live: {status}" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn only_the_author_or_an_admin_can_unshare(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let shared = share(&client, &base, ADMIN, "admin's plan").await?; + let id = shared["id"].as_str().unwrap(); + + let resp = client + .delete(format!("{base}/delete/{id}")) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 404); + let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM ai_shared_artifact") + .fetch_one(&db) + .await?; + assert_eq!(remaining, 1, "a member deleted someone else's share"); + + let member_share = share(&client, &base, MEMBER, "member's plan").await?; + let resp = client + .delete(format!( + "{base}/delete/{}", + member_share["id"].as_str().unwrap() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + Ok(()) +} + +/// The id is compared against a `VARCHAR(255)` column on every route that takes one, and a +/// NUL in it would otherwise reach Postgres and come back as a 500. +#[sqlx::test(fixtures("base"))] +async fn a_malformed_artifact_id_is_refused_on_every_route( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + for bad_id in ["", "a\0b", &"x".repeat(256)] { + let resp = client + .get(format!("{base}/status")) + .query(&[("artifact_id", bad_id)]) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 400, "status accepted {bad_id:?}"); + + let resp = client + .post(format!("{base}/share")) + .header("Authorization", MEMBER) + .json(&json!({ + "artifact_id": bad_id, + "name": "Plan", + "kind": "md", + "version": 1, + "content": "x", + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "share accepted {bad_id:?}"); + } + + Ok(()) +} diff --git a/backend/tests/app_guest_allowance.rs b/backend/tests/app_guest_allowance.rs new file mode 100644 index 0000000000..88626b1c81 --- /dev/null +++ b/backend/tests/app_guest_allowance.rs @@ -0,0 +1,159 @@ +//! The guest allowance: free up to `FREE_GUESTS_PER_WINDOW` distinct emails over the +//! trailing window. Past it, a hard-capped instance (Community, Pro) refuses a stranger +//! and lets a returning guest back in; a metered one (Enterprise) admits everyone and +//! counts seats. Its own binary: the plan is read from a process-wide key id that this +//! test flips, which no test sharing the process could tolerate. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::{FREE_GUESTS_PER_WINDOW, GUEST_WINDOW_DAYS}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +/// Community and Pro are capped, Enterprise is metered. Only a build with both +/// `private` (the key id) and `enterprise` (the plan read) can meter; every other build +/// is capped whatever this says. +fn set_plan(pro: bool) { + #[cfg(feature = "private")] + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new( + if pro { "test_pro" } else { "" }.to_string(), + )); + let _ = pro; +} + +async fn mint(db: &Pool, email: &str) -> windmill_common::error::Result { + let mut tx = db.begin().await.unwrap(); + let minted = windmill_api_users::users::create_guest_session_token( + email, + "test-workspace", + APP_PATH, + &mut tx, + tower_cookies::Cookies::default(), + ) + .await; + tx.commit().await.unwrap(); + minted +} + +#[sqlx::test(fixtures("base"))] +async fn the_allowance_caps_strangers_and_meters_an_enterprise_plan( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables_v2": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + // The whole allowance, used yesterday: still in the window, and a day the mint + // does not write, so a row dated today can only be the mint's own. + sqlx::query( + "INSERT INTO guest_activity (email, workspace_id, day) + SELECT 'g' || i || '@example.com', 'test-workspace', CURRENT_DATE - 1 + FROM generate_series(1, $1) AS i", + ) + .bind(FREE_GUESTS_PER_WINDOW) + .execute(&db) + .await?; + + set_plan(true); + let refused = mint(&db, "stranger@example.com").await.unwrap_err(); + assert!( + matches!(&refused, windmill_common::error::Error::PermissionDenied(m) + if m.contains(&format!("limit of {FREE_GUESTS_PER_WINDOW} guests over {GUEST_WINDOW_DAYS} days"))), + "a stranger past the allowance is refused with the message the visitor reads: {refused:?}" + ); + mint(&db, "g1@example.com") + .await + .expect("a guest already in the window is let back in"); + let recorded: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM guest_activity + WHERE email = 'g1@example.com' AND workspace_id = 'test-workspace' + AND day = CURRENT_DATE)", + ) + .fetch_one(&db) + .await?; + assert!( + recorded, + "the mint writes today's guest_activity row, the allowance's unit" + ); + + let list: serde_json::Value = authed( + client().get(format!( + "http://localhost:{port}/api/users/guests?per_page=5" + )), + ADMIN_TOKEN, + ) + .send() + .await? + .json() + .await?; + assert_eq!(list["usage"]["guest_count"], FREE_GUESTS_PER_WINDOW); + assert_eq!(list["usage"]["metered"], false); + assert_eq!(list["usage"]["guest_seats"], 0); + assert_eq!(list["guests"].as_array().map(Vec::len), Some(5)); + assert_eq!(list["guests"][0]["workspaces"], json!(["test-workspace"])); + let usage: serde_json::Value = authed( + client().get(format!("{ws}/workspaces/guest_usage")), + ADMIN_TOKEN, + ) + .send() + .await? + .json() + .await?; + assert_eq!(usage["guest_count"], FREE_GUESTS_PER_WINDOW); + + #[cfg(all(feature = "private", feature = "enterprise"))] + { + set_plan(false); + mint(&db, "stranger@example.com") + .await + .expect("a metered plan admits past the allowance"); + let usage: serde_json::Value = authed( + client().get(format!("{ws}/workspaces/guest_usage")), + ADMIN_TOKEN, + ) + .send() + .await? + .json() + .await?; + assert_eq!(usage["guest_count"], FREE_GUESTS_PER_WINDOW + 1); + assert_eq!(usage["metered"], true); + assert_eq!(usage["billable_guests"], 1); + assert_eq!( + usage["guest_seats"], 1, + "one guest past the allowance is a whole seat" + ); + } + + Ok(()) +} diff --git a/backend/tests/app_guest_cloud_hosted.rs b/backend/tests/app_guest_cloud_hosted.rs new file mode 100644 index 0000000000..7ab4145a65 --- /dev/null +++ b/backend/tests/app_guest_cloud_hosted.rs @@ -0,0 +1,168 @@ +//! Guests are unavailable on the shared cloud (`CLOUD_HOSTED`). +//! +//! One test in its own binary on purpose: `CLOUD_HOSTED` is read once into a +//! `lazy_static`, so it must be set before anything reads it and cannot be unset for a +//! sibling test in the same process. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const GUEST_TOKEN: &str = "GUEST_SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn the_cloud_admits_no_guest(db: Pool) -> anyhow::Result<()> { + // Before the server starts, so the flag is what the whole process sees. + unsafe { std::env::set_var("CLOUD_HOSTED", "true") }; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // The workspace switch cannot be turned on, so no policy can lean on it. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("self-hosted"), + "the refusal must name what guests need" + ); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "an app cannot be deployed to guests"); + + // Nor can a key be configured for the JWT way in — the refusal lands before the + // outbound JWKS fetch it would otherwise make. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({ "jwks_url": "https://issuer.example.com/.well-known/jwks.json" })) + .send() + .await?; + assert_eq!(resp.status(), 400, "a guest JWT key cannot be configured"); + // Clearing one stays allowed: a key nobody can use is still worth removing. + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An app already stored in guest mode — pushed by git-sync, or deployed before the + // instance became a cloud one — advertises no entry either. + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "UPDATE app SET policy = jsonb_set(policy, '{execution_mode}', '\"guest\"') + WHERE path = $1 AND workspace_id = 'test-workspace'", + ) + .bind(APP_PATH) + .execute(&db) + .await?; + sqlx::query("UPDATE workspace_settings SET guest_access_enabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + + // Deploying it again is not refused: only widening an app into guests is, so a + // git-sync push of one already stored that way keeps working (and keeps being inert). + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "an app already stored in guest mode must stay deployable: {}", + resp.text().await? + ); + + let resp = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "reading the share secret must succeed"); + let secret: String = resp.text().await?; + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a guest app must not advertise entry where guests are unavailable" + ); + + // And a session issued before the instance became a cloud one stops on its next + // request: the door re-reads the switch, so the credential itself is not enough. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'GUEST_SECR', $2, 'guest@example.com', + 'guest_session', $3, 'test-workspace', now() + interval '8 hours')", + ) + .bind(GUEST_TOKEN.as_bytes()) + .bind(GUEST_TOKEN) + .bind(vec![ + "guest".to_string(), + "users:read".to_string(), + format!("apps:read:{APP_PATH}"), + format!("apps:run:{APP_PATH}"), + ]) + .execute(&db) + .await?; + // `whoami` is where an admitted guest resolves as `role: guest`, so a 401 here is + // the door refusing the credential rather than a route saying no. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest session must not authenticate where guests are unavailable" + ); + + Ok(()) +} diff --git a/backend/tests/app_guest_execution_mode.rs b/backend/tests/app_guest_execution_mode.rs new file mode 100644 index 0000000000..7af2e0794e --- /dev/null +++ b/backend/tests/app_guest_execution_mode.rs @@ -0,0 +1,1092 @@ +//! Tests for the `guest` app execution mode. +//! +//! A guest (`ExecutionMode::Guest`) has no account and so no ACL of its own: its +//! token's scopes are its entire grant. These tests pin the three things that would +//! silently undo it: +//! +//! * what makes a token a guest — the server-minted label, never a scope anyone +//! could type into `users/tokens/create`; +//! * the confinement — a guest reaches the one app it was let in for and nothing +//! else; +//! * the switches — an app's own `execution_mode: guest` is inert unless the +//! workspace and the instance allow guests, checked at the door rather than only +//! where a policy is written (git-sync and the CLI push policies past every UI); +//! the allowance on top of them has a binary of its own. +//! +//! The token is inserted directly: how a guest session is minted is the identity +//! provider's business (EE), what one can do is this file's. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const GUEST_TOKEN: &str = "GUEST_SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +async fn enable_guests(port: u16, ws: &str) -> anyhow::Result<()> { + authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/workspaces/edit_guest_access" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + Ok(()) +} + +fn guest_scopes() -> Vec { + vec![ + "guest".to_string(), + "jobs:read".to_string(), + "resources:run".to_string(), + "users:read".to_string(), + "folders:read".to_string(), + format!("apps:read:{APP_PATH}"), + format!("apps:run:{APP_PATH}"), + ] +} + +/// Insert a guest session for `test-workspace`, scoped to `APP_PATH`. Mirrors +/// `create_guest_session_token`: the server-minted label, the narrow reads, the two +/// path-scoped app grants, the workspace pin, and an expiry — a derived token's +/// lifetime is capped at it, so a guest session without one cannot mint. +async fn insert_guest_token(db: &Pool, workspace: &str) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'GUEST_SECR', $2, 'guest@example.com', + 'guest_session', $3, $4, now() + interval '8 hours')", + ) + .bind(GUEST_TOKEN.as_bytes()) + .bind(GUEST_TOKEN) + .bind(guest_scopes()) + .bind(workspace) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn guest_session_is_confined_to_its_app(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + insert_guest_token(&db, "test-workspace").await?; + + // Its own identity resolves, and reports the role rather than falling through to + // the non-member branch that hands out a `superadmin` shape. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "guest whoami must resolve"); + let me: serde_json::Value = resp.json().await?; + assert_eq!( + me["role"], + json!("guest"), + "guest must not read as superadmin" + ); + assert_eq!(me["operator"], json!(true)); + assert_eq!(me["is_admin"], json!(false)); + + // `resources/list_names` and the type schemas stay open — a guest drives an app, + // and app pickers need them — so the line to pin is the value-returning route. + for route in [ + "jobs/list", + "scripts/list", + "flows/list", + "variables/list", + "resources/get_value/u/test-user/secret", + "apps/list", + ] { + let resp = authed(client().get(format!("{ws}/{route}")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "guest must be denied {route}, got {}", + resp.status() + ); + } + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn guest_token_does_not_cross_workspaces(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // A second workspace with an app at the SAME path: without the token's workspace + // pin, `apps:run:` would unlock it too, since a path is not unique across + // workspaces. + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ('other-ws', 'other-ws', 'test-user')", + ) + .execute(&db) + .await?; + sqlx::query("INSERT INTO workspace_settings (workspace_id) VALUES ('other-ws')") + .execute(&db) + .await?; + + insert_guest_token(&db, "test-workspace").await?; + + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/other-ws/apps/get/p/{APP_PATH}" + )), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest token pinned to one workspace must not authenticate against another" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn guest_entry_needs_both_the_app_mode_and_the_workspace_switch( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + + let secret: String = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + + // The app says guest, the workspace has not opted in: inert. + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a guest app in a workspace that has not enabled guests must not advertise entry" + ); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + + // Unauthenticated on purpose: this is what a signed-out visitor reads. + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let entry: serde_json::Value = resp.json().await?; + assert_eq!(entry["app_path"], json!(APP_PATH)); + + // Turning the switch back off closes the door again even though the app's own + // policy is unchanged. + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": false })) + .send() + .await?; + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "turning guests off must stop advertising entry for an app already set to guest" + ); + + Ok(()) +} + +/// The guest grant is the server-minted label, never the `guest` scope. Scopes on a +/// user-created token are whatever the caller typed, so if the scope granted anything +/// then any member of any workspace could mint themselves non-member access to every +/// guest-mode app on the instance. +#[sqlx::test(fixtures("base"))] +async fn a_self_declared_guest_scope_grants_nothing(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + // `users/tokens/create` must refuse the label outright... + let resp = authed( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "guest_session", "scopes": guest_scopes() })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "the guest session label must be server-minted only" + ); + + // ...and so must relabelling an ordinary token into it, or the pin-less user + // token would become a guest session that authenticates in every workspace. + let resp = authed( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "mine", "scopes": guest_scopes() })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let prefix: String = sqlx::query_scalar( + "SELECT token_prefix FROM token WHERE email = 'test@windmill.dev' AND label = 'mine'", + ) + .fetch_one(&db) + .await?; + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/tokens/update_label/{prefix}" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "guest_session" })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "relabelling into the guest namespace must be refused: {}", + resp.text().await? + ); + + // ...and a token that carries the scopes under any other label authenticates as + // nothing in a workspace its owner is not a member of. + // An email with no `usr` row anywhere: exactly the identity the guest arm exists + // to admit, and the one a forged scope must not admit. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes) + VALUES (encode(sha256($1::bytea), 'hex'), 'FORGED_SCO', $2, 'outsider@example.com', + 'forged', $3)", + ) + .bind(b"FORGED_SCOPES".as_slice()) + .bind("FORGED_SCOPES") + .bind(guest_scopes()) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{ws}/users/whoami")), "FORGED_SCOPES") + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "declaring the guest scope must not turn a non-member into an identity" + ); + + Ok(()) +} + +/// A guest-mode policy that names one runnable, so an `execute_component` request +/// gets past the triggerables lookup and reaches the guest gate. `sandbox` is what +/// makes the embed-token endpoint actually mint a token. +fn guest_app_with_runnable(path: &str, sandbox: bool) -> serde_json::Value { + app_with_runnable(path, "guest", sandbox) +} + +fn app_with_runnable(path: &str, execution_mode: &str, sandbox: bool) -> serde_json::Value { + json!({ + "path": path, + "summary": "App", + "value": {}, + "policy": { + "execution_mode": execution_mode, + "sandbox": sandbox, + "triggerables_v2": { + "script/u/test-user/noop": { "static_inputs": {}, "one_of_inputs": {} } + } + } + }) +} + +fn execute(port: u16, ws: &str, app: &str, token: &str) -> reqwest::RequestBuilder { + authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/apps_u/execute_component/{app}" + )), + token, + ) + .json(&json!({ + "component": "a", + "path": "script/u/test-user/noop", + "args": {} + })) +} + +/// The workspace switch is enforced at the auth door for every guest request, not +/// remembered per handler. This is what stands between a `guest` policy pushed by +/// git-sync and execution once an admin has turned guests off — and it closes the +/// app to sessions already issued. +#[sqlx::test(fixtures("base"))] +async fn the_door_re_checks_the_workspace_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; + + // Switch off: the session does not authenticate at all, even though the app's + // policy says guest and the session was (in this fixture) issued regardless. On + // the authed route that is a 401; on the optional-auth run route the rejected + // token reads as no token, and a guest-mode app then refuses the anonymous + // caller — a denial either way. + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest must not authenticate while guests are off" + ); + let resp = execute(port, "test-workspace", APP_PATH, GUEST_TOKEN) + .send() + .await?; + assert!( + resp.status().is_client_error() && resp.status() != 404, + "a guest must not run while guests are off, got {}", + resp.status() + ); + + // Switch on: through the door. What follows the run is the runnable lookup, + // which fails on the nonexistent script — the point is that it is no longer a + // denial. + enable_guests(port, "test-workspace").await?; + let resp = execute(port, "test-workspace", APP_PATH, GUEST_TOKEN) + .send() + .await?; + assert!( + resp.status() != 401 && resp.status() != 403, + "with guests on, the door must let the run through: {}", + resp.status() + ); + + Ok(()) +} + +/// The path scope is what keeps a guest to the one app it was let in for: the route +/// layer is resource-blind for `apps:run`, so this line is drawn in the handler. +#[sqlx::test(fixtures("base"))] +async fn guest_cannot_run_another_guest_app(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let other = "u/test-user/other_guest_app"; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(other, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; // scoped to APP_PATH, not `other` + + let resp = execute(port, "test-workspace", other, GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "a guest session scoped to one app must not run another, even one open to guests" + ); + + Ok(()) +} + +/// The app path is spliced into the session's scopes, whose grammar reserves `:`, `,` +/// and `*`: a path carrying one would scope the guest to more than the one app it was +/// let in for, so the mint refuses it before anything else. Anything else in a path +/// (spaces, `@`) is literal to that grammar and stays admissible. +#[sqlx::test(fixtures("base"))] +async fn a_scope_metacharacter_in_the_app_path_is_refused( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let mint = |path: &'static str| { + let db = db.clone(); + async move { + let mut tx = db.begin().await?; + let minted = windmill_api_users::users::create_guest_session_token( + "guest@example.com", + "test-workspace", + path, + &mut tx, + tower_cookies::Cookies::default(), + ) + .await; + anyhow::Ok(minted) + } + }; + for path in [ + "u/test-user/entry,u/test-user/hidden", + "u/test-user/*", + "u/test-user/entry:run", + ] { + let minted = mint(path).await?; + assert!( + matches!(minted, Err(windmill_common::error::Error::BadRequest(ref m)) if m.contains("cannot be scoped")), + "{path}: {minted:?}" + ); + } + for path in ["u/test-user/My App", "u/admin@windmill.dev/x"] { + let minted = mint(path).await?; + assert!( + !matches!(minted, Err(windmill_common::error::Error::BadRequest(ref m)) if m.contains("cannot be scoped")), + "{path} is literal to the scope grammar and must get past the guard: {minted:?}" + ); + } + Ok(()) +} + +/// A guest reads the jobs it launched and nothing else: with no membership behind it, +/// it must stop where an app embed token stops, before the share-token and ACL grants +/// a member would get, and with the same "not found" so it cannot probe for jobs. +#[sqlx::test(fixtures("base"))] +async fn a_guest_cannot_read_a_job_it_did_not_launch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + insert_guest_token(&db, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/scripts/create")), ADMIN_TOKEN) + .json(&json!({ + "path": "u/test-user/noop", + "summary": "", + "description": "", + "content": "echo 42", + "language": "bash", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = authed( + client().post(format!("{ws}/jobs/run/p/u/test-user/noop")), + ADMIN_TOKEN, + ) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let job_id = resp.text().await?; + + let resp = authed( + client().get(format!("{ws}/jobs_u/getupdate/{job_id}")), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "another caller's job is not found for a guest: {}", + resp.text().await? + ); + + Ok(()) +} + +/// Guests mode cannot land on a path the scope grammar cannot hold, however it gets +/// there: set at creation, set on update, or a rename of an app already in that mode. +#[sqlx::test(fixtures("base"))] +async fn guests_mode_needs_a_scopable_path(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable("u/test-user/a:b", false)) + .send() + .await?; + assert_eq!(resp.status(), 400, "created into Guests on a `:` path"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ "path": "u/test-user/a,b" })) + .send() + .await?; + assert_eq!(resp.status(), 400, "renamed to a `,` path while in Guests"); + let resp = authed( + client().post(format!("{ws}/apps/update/{APP_PATH}")), + ADMIN_TOKEN, + ) + .json(&json!({ "path": "u/test-user/My App" })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a space is literal: {}", + resp.text().await? + ); + + // Set on update: an app that already sits on such a path cannot be switched. + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": "u/test-user/x:y", + "summary": "App", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables_v2": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let resp = authed( + client().post(format!("{ws}/apps/update/u/test-user/x:y")), + ADMIN_TOKEN, + ) + .json(&json!({ "policy": { "execution_mode": "guest", "triggerables_v2": {} } })) + .send() + .await?; + assert_eq!(resp.status(), 400, "switched to Guests on a `:` path"); + + Ok(()) +} + +/// Renaming a workspace copies its settings; the guest switch and the guest JWT key must +/// travel with them, or the rename silently shuts every guest app or drops the key. +#[sqlx::test(fixtures("base"))] +async fn a_workspace_rename_keeps_the_guest_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + enable_guests(port, "test-workspace").await?; + sqlx::query( + "INSERT INTO guest_activity (email, workspace_id, day) + VALUES ('guest@example.com', 'test-workspace', CURRENT_DATE)", + ) + .execute(&db) + .await?; + sqlx::query( + "UPDATE workspace_settings SET guest_jwt_public_key = 'test-pem-key' WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/change_workspace_id" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "new_id": "test-workspace-2", "new_name": "Test workspace 2" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let enabled: bool = sqlx::query_scalar( + "SELECT guest_access_enabled FROM workspace_settings WHERE workspace_id = 'test-workspace-2'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "the guest switch travels with the workspace"); + let jwt_key: Option = sqlx::query_scalar( + "SELECT guest_jwt_public_key FROM workspace_settings WHERE workspace_id = 'test-workspace-2'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + jwt_key.as_deref(), + Some("test-pem-key"), + "the guest JWT key travels with the workspace" + ); + let moved: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM guest_activity WHERE workspace_id = 'test-workspace-2') + AND NOT EXISTS(SELECT 1 FROM guest_activity WHERE workspace_id = 'test-workspace')", + ) + .fetch_one(&db) + .await?; + assert!(moved, "the guests seen in the workspace follow its new id"); + + Ok(()) +} + +/// The superadmin switch sits above every workspace's: off, no guest session stands and +/// no app discovers as open, whatever the workspace and the app say. +#[sqlx::test(fixtures("base"))] +async fn the_instance_switch_closes_every_workspace(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; + let set_instance_switch = |disabled: bool| { + authed( + client().post(format!( + "http://localhost:{port}/api/settings/global/guest_access_disabled" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "value": disabled })) + .send() + }; + + let secret: String = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + + set_instance_switch(true).await?.error_for_status()?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "the instance switch closes an issued session" + ); + let resp = client() + .get(format!("{ws}/apps_u/guest_entry/{secret}")) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "and nothing discovers as open to guests" + ); + + set_instance_switch(false).await?.error_for_status()?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "back on, the session stands again"); + + Ok(()) +} + +/// An account holder is never a guest, and that holds after the mint too: a session +/// minted before the account existed ends at the door the moment one does, so an +/// account provisioned in a race with the mint cannot outlive the rule. +#[sqlx::test(fixtures("base"))] +async fn an_account_ends_the_guest_session(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + insert_guest_token(&db, "test-workspace").await?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "guest whoami must resolve"); + + sqlx::query( + "INSERT INTO password (email, password_hash, login_type, super_admin, verified, name) + VALUES ('guest@example.com', 'not-a-real-hash', 'password', false, true, 'Guest')", + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{ws}/users/whoami")), GUEST_TOKEN) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "an account created after the mint ends the guest session at the door" + ); + + Ok(()) +} + +/// An upload goes through an app's `s3_inputs` policy or not at all for a guest: the +/// legacy branch for an app without one uploads with the caller's own standing, which a +/// guest has none of, and an app path with no row must not slip past the confinement. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn a_guest_cannot_upload_outside_a_policy(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) // no `s3_inputs` + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; + + let upload = |app: &str| { + authed( + client().post(format!( + "{ws}/apps_u/upload_s3_file/{app}?file_key=anything" + )), + GUEST_TOKEN, + ) + .body("x") + .send() + }; + let resp = upload("u/test-user/no_such_app").await?; + assert_eq!( + resp.status(), + 403, + "a path with no app must not escape the guest's confinement: {}", + resp.text().await? + ); + let resp = upload(APP_PATH).await?; + assert_eq!( + resp.status(), + 400, + "without an upload policy a guest is refused like an anonymous caller: {}", + resp.text().await? + ); + + Ok(()) +} + +/// An anonymous app is open to anyone, a guest included, and the guest uses it as +/// itself: the component run and the result read that follows are one identity, so +/// the read's launched-by-me grant matches. Acting as nobody for the run and as the +/// guest for the read would start a job whose result the page can never fetch. +#[sqlx::test(fixtures("base"))] +async fn a_guest_uses_an_anonymous_app_as_itself(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/scripts/create")), ADMIN_TOKEN) + .json(&json!({ + "path": "u/test-user/noop", + "summary": "", + "description": "", + "content": "echo 42", + "language": "bash", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let anon = "u/test-user/anon_app"; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&app_with_runnable(anon, "anonymous", false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + insert_guest_token(&db, "test-workspace").await?; // scoped to APP_PATH, not `anon` + + let resp = execute(port, "test-workspace", anon, GUEST_TOKEN) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let job_id = resp.text().await?; + + let resp = authed( + client().get(format!("{ws}/jobs_u/getupdate/{job_id}")), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "the guest that started the run must be able to read it back: {}", + resp.text().await? + ); + + Ok(()) +} + +/// The embed token a guest mints for a sandboxed app is the one credential handed to +/// untrusted app JS. It must be a guest twice over — resolve like its minter (the +/// label) and be governed like its minter (the sentinel) — or every guest control +/// silently skips the most exposed credential there is. +#[sqlx::test(fixtures("base"))] +async fn a_guest_minted_embed_token_stays_a_guest(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, true)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + let secret: String = authed( + client().get(format!("{ws}/apps/secret_of/{APP_PATH}")), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + insert_guest_token(&db, "test-workspace").await?; + + // The guest page mints the iframe's token from the guest session. + let resp = authed( + client().get(format!("{ws}/apps_u/embed_token/{secret}")), + GUEST_TOKEN, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a guest must be able to mint: {}", + resp.text().await? + ); + let body: serde_json::Value = resp.json().await?; + let embed = body["token"] + .as_str() + .expect("mint must return a token for an authenticated guest") + .to_string(); + + // Its lifetime is capped at the session that minted it: the requested embed + // validity (12h) is longer than the guest session's (8h in this fixture), and the + // session's expiry is a guest's only revocation. + let parent_exp: chrono::DateTime = + sqlx::query_scalar("SELECT expiration FROM token WHERE token_prefix = 'GUEST_SECR'") + .fetch_one(&db) + .await?; + let child_exp: chrono::DateTime = body["expiration"] + .as_str() + .and_then(|e| e.parse().ok()) + .expect("mint must return the token's expiration"); + assert!( + child_exp <= parent_exp, + "a guest's embed token must not outlive the session that minted it ({child_exp} > {parent_exp})" + ); + + // Resolves — and as a guest, not as the non-member superadmin shape. + let resp = authed(client().get(format!("{ws}/users/whoami")), &embed) + .send() + .await?; + assert_eq!(resp.status(), 200, "the minted token must authenticate"); + let me: serde_json::Value = resp.json().await?; + assert_eq!(me["role"], json!("guest")); + + // Governed: the workspace switch closes it at the door, iframe or not. + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": false })) + .send() + .await?; + let resp = authed(client().get(format!("{ws}/users/whoami")), &embed) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "turning guests off must stop a guest's embed token authenticating" + ); + let resp = execute(port, "test-workspace", APP_PATH, &embed) + .send() + .await?; + assert!( + resp.status().is_client_error() && resp.status() != 404, + "and running components, got {}", + resp.status() + ); + enable_guests(port, "test-workspace").await?; + + // And its scopes are not something the guest's email can later rewrite. The + // guest session itself cannot reach `/users/*` (workspace pin), so model the real + // threat: the same email after promotion, holding an ordinary unpinned session. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label) + VALUES (encode(sha256($1::bytea), 'hex'), 'PROMOTED_S', $2, 'guest@example.com', + 'session')", + ) + .bind(b"PROMOTED_SESSION".as_slice()) + .bind("PROMOTED_SESSION") + .execute(&db) + .await?; + for prefix in [&embed[..10], &GUEST_TOKEN[..10]] { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/tokens/update_scopes/{prefix}" + )), + "PROMOTED_SESSION", + ) + .json(&json!({ "scopes": null })) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a promoted account must not be able to rescope its old guest credentials" + ); + } + + Ok(()) +} + +/// The label is the single source of truth: a guest-labelled credential is governed +/// as a guest even if its scopes carry no sentinel. Otherwise every mint that derives +/// a token from a guest session is one forgotten `push` away from an ungoverned +/// non-member credential. +#[sqlx::test(fixtures("base"))] +async fn a_guest_label_is_governed_without_the_sentinel(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + enable_guests(port, "test-workspace").await?; + let scopes: Vec = guest_scopes() + .into_iter() + .filter(|s| s != "guest") + .collect(); + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes, workspace_id, expiration) + VALUES (encode(sha256($1::bytea), 'hex'), 'NOSENTINE_', $2, 'guest@example.com', + 'guest_session', $3, 'test-workspace', now() + interval '8 hours')", + ) + .bind(b"NOSENTINEL".as_slice()) + .bind("NOSENTINEL") + .bind(scopes) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{ws}/users/whoami")), "NOSENTINEL") + .send() + .await?; + assert_eq!(resp.status(), 200); + let me: serde_json::Value = resp.json().await?; + assert_eq!( + me["role"], + json!("guest"), + "the label alone must make a credential a guest" + ); + let resp = authed(client().get(format!("{ws}/jobs/list")), "NOSENTINEL") + .send() + .await?; + assert_eq!(resp.status(), 403, "and confine it like one"); + + Ok(()) +} + +/// A guest is someone with no account at all — including a deactivated one. The +/// sign-in path's own account lookup filters on `disabled = false`, so a disabled +/// account reads as absent there; the mint has to refuse on its own or deactivation +/// (manual or SCIM, whose revocation is "delete the tokens") walks straight back in. +#[sqlx::test(fixtures("base"))] +async fn a_disabled_account_cannot_become_a_guest(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&guest_app_with_runnable(APP_PATH, false)) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "INSERT INTO password (email, password_hash, login_type, super_admin, verified, disabled) + VALUES ('gone@example.com', 'x', 'password', false, true, true)", + ) + .execute(&db) + .await?; + + let mut tx = db.begin().await?; + let cookies = tower_cookies::Cookies::default(); + let minted = windmill_api_users::users::create_guest_session_token( + "gone@example.com", + "test-workspace", + APP_PATH, + &mut tx, + cookies, + ) + .await; + assert!( + matches!(minted, Err(windmill_common::error::Error::NotAuthorized(_))), + "a deactivated account must be refused a guest session, got {minted:?}" + ); + + Ok(()) +} diff --git a/backend/tests/app_guest_jwt_allowance.rs b/backend/tests/app_guest_jwt_allowance.rs new file mode 100644 index 0000000000..85cb51d39e --- /dev/null +++ b/backend/tests/app_guest_jwt_allowance.rs @@ -0,0 +1,128 @@ +//! The guest allowance reached through a guest JWT (`jwt_guest_`). Its own binary +//! because `set_plan` flips a process-global license key, which a test sharing the +//! process could not tolerate (see `app_guest_allowance.rs`). +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::FREE_GUESTS_PER_WINDOW; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +/// Community and Pro are capped, Enterprise is metered. Only a build with both +/// `private` and `enterprise` can meter; every other build is capped whatever this says. +fn set_plan(pro: bool) { + #[cfg(feature = "private")] + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new( + if pro { "test_pro" } else { "" }.to_string(), + )); + let _ = pro; +} + +const JWT_PUB: &str = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzAfqyCh34iYOCW0vg4ejq/zzJlzL\nSZScjnVyPjLGTapEwo4gc6/y1Yudd/v54wKh0OdfTfzAKMPWx/2NWx/ugg==\n-----END PUBLIC KEY-----\n"; +const JWT_PRIV: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgu27S2DbSwUh8BmQb\n/i4/VhNdoXV7PJekhnoceMULYLihRANCAATMB+rIKHfiJg4JbS+Dh6Or/PMmXMtJ\nlJyOdXI+MsZNqkTCjiBzr/LVi513+/njAqHQ519N/MAow9bH/Y1bH+6C\n-----END PRIVATE KEY-----\n"; + +fn guest_jwt(email: &str) -> String { + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + let exp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + + 3600; + let claims = json!({ + "email": email, + "workspace_id": "test-workspace", + "app_path": APP_PATH, + "exp": exp, + }); + let jwt = encode( + &Header::new(Algorithm::ES256), + &claims, + &EncodingKey::from_ec_pem(JWT_PRIV.as_bytes()).unwrap(), + ) + .unwrap(); + format!("jwt_guest_{jwt}") +} + +/// A JWT guest is subject to the same allowance as a signed-in one. Past the cap on a +/// capped instance, a stranger's JWT is refused (the auth arm returns 401; the visitor +/// message is only logged, since the arm cannot carry it), while a guest already in the +/// window is let back in. +#[sqlx::test(fixtures("base"))] +async fn a_guest_jwt_is_capped_like_a_signed_in_guest(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + authed( + client().post(format!("{ws}/workspaces/edit_guest_access")), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": true })) + .send() + .await?; + let resp = authed( + client().post(format!("{ws}/workspaces/edit_guest_jwt_key")), + ADMIN_TOKEN, + ) + .json(&json!({ "public_key": JWT_PUB })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "Guest app", + "value": {}, + "policy": { "execution_mode": "guest", "triggerables_v2": {} } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + // The whole allowance, used today (g1..gN). + sqlx::query( + "INSERT INTO guest_activity (email, workspace_id, day) + SELECT 'g' || i || '@example.com', 'test-workspace', CURRENT_DATE + FROM generate_series(1, $1) AS i", + ) + .bind(FREE_GUESTS_PER_WINDOW) + .execute(&db) + .await?; + set_plan(true); + + let resp = authed( + client().get(format!("{ws}/users/whoami")), + &guest_jwt("stranger@example.com"), + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "a stranger's JWT is refused past the cap"); + + let resp = authed( + client().get(format!("{ws}/users/whoami")), + &guest_jwt("g1@example.com"), + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a returning guest's JWT is admitted: {}", + resp.text().await? + ); + + Ok(()) +} diff --git a/backend/tests/app_guest_jwt_entry.rs b/backend/tests/app_guest_jwt_entry.rs new file mode 100644 index 0000000000..7b84130a04 --- /dev/null +++ b/backend/tests/app_guest_jwt_entry.rs @@ -0,0 +1,494 @@ +//! Tests for the guest JWT entry: a guest that enters through a JWT the embedding +//! customer's own backend mints and signs, with no identity-provider round-trip. +//! +//! The key is a per-workspace setting (a PEM public key here), and the token is +//! verified per request against it. A JWT guest is the same identity as a signed-in +//! guest: no `usr` row, no `password` row, no seat, confined to the one app its +//! `app_path` names. These tests pin what a token must carry to be honoured, and the +//! refusals that keep the door narrow: wrong workspace, wrong key, expired, a +//! symmetric algorithm, an email that already has an account, an app not in guest +//! mode, and the workspace switch off. +//! +//! The keys are fixed test vectors (EC P-256, PKCS8), so signing is deterministic and +//! needs no key generation at runtime. + +// Built with these like the sibling guest-execution suite: the guest run executes as +// the publisher through EE on-behalf-of code. CI builds with them. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use std::time::{SystemTime, UNIX_EPOCH}; + +use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; +use serde::Serialize; +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const APP_PATH: &str = "u/test-user/guest_app"; +const GUEST_EMAIL: &str = "guest@example.com"; + +// A P-256 keypair the workspace verifies against (PUB1), and a second private key +// (PRIV2) that it does not, for the wrong-key refusal. +const PRIV1: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgu27S2DbSwUh8BmQb\n/i4/VhNdoXV7PJekhnoceMULYLihRANCAATMB+rIKHfiJg4JbS+Dh6Or/PMmXMtJ\nlJyOdXI+MsZNqkTCjiBzr/LVi513+/njAqHQ519N/MAow9bH/Y1bH+6C\n-----END PRIVATE KEY-----\n"; +const PUB1: &str = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzAfqyCh34iYOCW0vg4ejq/zzJlzL\nSZScjnVyPjLGTapEwo4gc6/y1Yudd/v54wKh0OdfTfzAKMPWx/2NWx/ugg==\n-----END PUBLIC KEY-----\n"; +const PRIV2: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgjyhWYyI2+z5zTT0B\neI9EuJJ7v0tcNXhvHrq9y2AG1LihRANCAAS40dEdO+tTffhGt4YQv0dStkd6VcWN\n+CHI9QqZAHAJMsNS3Ld+sZe2M6Of0CNR300QJtfp4UIdEVbXBCIxL1D0\n-----END PRIVATE KEY-----\n"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +#[derive(Serialize)] +struct Claims { + email: String, + workspace_id: String, + app_path: String, + exp: u64, + #[serde(skip_serializing_if = "Option::is_none")] + nbf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + iat: Option, +} + +impl Claims { + fn valid() -> Self { + Claims { + email: GUEST_EMAIL.to_string(), + workspace_id: "test-workspace".to_string(), + app_path: APP_PATH.to_string(), + exp: now() + 3600, + nbf: None, + iat: None, + } + } +} + +/// Sign as a bearer (`jwt_guest_`). `priv_pem`/`alg` let a test sign with the +/// wrong key or a refused algorithm. +fn bearer(claims: &Claims, priv_pem: &str, alg: Algorithm) -> String { + let key = match alg { + Algorithm::HS256 => EncodingKey::from_secret(b"a-shared-secret"), + _ => EncodingKey::from_ec_pem(priv_pem.as_bytes()).unwrap(), + }; + let jwt = encode(&Header::new(alg), claims, &key).unwrap(); + format!("jwt_guest_{jwt}") +} + +async fn enable_guests(port: u16, ws: &str, on: bool) -> anyhow::Result<()> { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/workspaces/edit_guest_access" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "guest_access_enabled": on })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +async fn set_guest_jwt_pem(port: u16, ws: &str, pem: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/workspaces/edit_guest_jwt_key" + )), + ADMIN_TOKEN, + ) + .json(&json!({ "public_key": pem })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +fn app(path: &str, execution_mode: &str, sandbox: bool) -> serde_json::Value { + json!({ + "path": path, + "summary": "App", + "value": {}, + "policy": { + "execution_mode": execution_mode, + "sandbox": sandbox, + "triggerables_v2": { + "script/u/test-user/noop": { "static_inputs": {}, "one_of_inputs": {} } + } + } + }) +} + +async fn create_app(port: u16, ws: &str, v: serde_json::Value) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("http://localhost:{port}/api/w/{ws}/apps/create")), + ADMIN_TOKEN, + ) + .json(&v) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + Ok(()) +} + +fn whoami(port: u16, ws: &str, token: &str) -> reqwest::RequestBuilder { + authed( + client().get(format!("http://localhost:{port}/api/w/{ws}/users/whoami")), + token, + ) +} + +/// A valid guest JWT opens its app, runs a component as the publisher, reads the run +/// back, reports `role: guest`, and leaves exactly one `guest_activity` row however +/// many requests it makes. +#[sqlx::test(fixtures("base"))] +async fn a_valid_guest_jwt_opens_its_app(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + let resp = authed( + client().post(format!("http://localhost:{port}/api/w/{ws}/scripts/create")), + ADMIN_TOKEN, + ) + .json(&json!({ + "path": "u/test-user/noop", + "summary": "", + "description": "", + "content": "echo 42", + "language": "bash", + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + + // A distinct email: the activity write is deduplicated by a process-global cache + // keyed on email, workspace and day, and other tests in this binary share the + // guest email, so the count below is only this test's if its email is its own. + let mut claims = Claims::valid(); + claims.email = "activity-guest@example.com".to_string(); + let token = bearer(&claims, PRIV1, Algorithm::ES256); + + let resp = whoami(port, ws, &token).send().await?; + assert_eq!(resp.status(), 200, "guest JWT must authenticate"); + let me: serde_json::Value = resp.json().await?; + assert_eq!(me["role"], json!("guest"), "must read as a guest"); + assert_eq!(me["operator"], json!(true)); + assert_eq!(me["is_admin"], json!(false)); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{ws}/apps_u/execute_component/{APP_PATH}" + )), + &token, + ) + .json(&json!({ "component": "a", "path": "script/u/test-user/noop", "args": {} })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let job_id = resp.text().await?; + + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/{ws}/jobs_u/getupdate/{job_id}" + )), + &token, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "the guest that started the run must read it back: {}", + resp.text().await? + ); + + // Several requests, one row: the write is cached per email, workspace and day. + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM guest_activity WHERE email = $1 AND workspace_id = $2 AND jwt_entry", + ) + .bind(&claims.email) + .bind(ws) + .fetch_one(&db) + .await?; + assert_eq!(count, 1, "a JWT guest must leave exactly one activity row"); + + Ok(()) +} + +/// The refusals that keep the door narrow. Each presents a bearer on the workspace's +/// own `whoami`, which the arm reaches only after every gate, so a 401 is the arm +/// saying no rather than a handler. +#[sqlx::test(fixtures("base"))] +async fn guest_jwt_refusals(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + create_app(port, ws, app("u/test-user/members_app", "publisher", false)).await?; + + // Positive control: a token valid against this exact fixture is admitted. Without it a + // broken setup would 401 every bearer below and the whole suite would pass vacuously. + let control = whoami(port, ws, &bearer(&Claims::valid(), PRIV1, Algorithm::ES256)) + .send() + .await?; + assert_eq!(control.status(), 200, "{}", control.text().await?); + + // wrong workspace: the claim must name the route's workspace. + let mut c = Claims::valid(); + c.workspace_id = "other-ws".to_string(); + let wrong_ws = bearer(&c, PRIV1, Algorithm::ES256); + + // wrong key: signed with a key the workspace does not hold. + let wrong_key = bearer(&Claims::valid(), PRIV2, Algorithm::ES256); + + // expired, past the verifier's clock-skew leeway. + let mut c = Claims::valid(); + c.exp = now() - 120; + let expired = bearer(&c, PRIV1, Algorithm::ES256); + + // a symmetric algorithm is never accepted. + let hs256 = bearer(&Claims::valid(), PRIV1, Algorithm::HS256); + + // an email that already has an account is refused, not downgraded. + let mut c = Claims::valid(); + c.email = "test@windmill.dev".to_string(); + let has_account = bearer(&c, PRIV1, Algorithm::ES256); + + // an app not in guest mode. + let mut c = Claims::valid(); + c.app_path = "u/test-user/members_app".to_string(); + let not_guest_app = bearer(&c, PRIV1, Algorithm::ES256); + + // an existing account addressed in a different case still counts as an account: + // the base fixture holds `test@windmill.dev`. + let mut c = Claims::valid(); + c.email = "Test@Windmill.Dev".to_string(); + let mixed_case_account = bearer(&c, PRIV1, Algorithm::ES256); + + // a lifetime past the 24h cap, even with a valid signature. + let mut c = Claims::valid(); + c.exp = now() + 25 * 3600; + let over_lifetime_cap = bearer(&c, PRIV1, Algorithm::ES256); + + // an email with no `@` would become the guest's username and could be read as a + // `u/` or `g/` principal; refused. + let mut c = Claims::valid(); + c.email = "group-admins".to_string(); + let group_shaped_email = bearer(&c, PRIV1, Algorithm::ES256); + + // an email longer than the `guest_activity.email` column: refused before auth, so a + // guest is never admitted without the activity row and audit event the count needs. + let mut c = Claims::valid(); + c.email = format!("{}@example.com", "a".repeat(250)); + let oversized_email = bearer(&c, PRIV1, Algorithm::ES256); + + // an app_path carrying a scope metacharacter would widen the guest's scopes. + let mut c = Claims::valid(); + c.app_path = "u/test-user/*".to_string(); + let wildcard_app_path = bearer(&c, PRIV1, Algorithm::ES256); + + // a valid, signed token past the length cap: without the cap it would deserialize into + // GuestJwtClaims (the extra claim ignored) and verify, so this pins the length check. + let mut payload = serde_json::to_value(Claims::valid()).unwrap(); + payload["padding"] = serde_json::json!("a".repeat(9000)); + let big_jwt = encode( + &Header::new(Algorithm::ES256), + &payload, + &EncodingKey::from_ec_pem(PRIV1.as_bytes()).unwrap(), + ) + .unwrap(); + let oversized_token = format!("jwt_guest_{big_jwt}"); + + // a repeated prefix must not strip down to a valid short token that verifies and is then + // cached under the full bearer key (trim_start_matches would; strip_prefix must not). + let repeated_prefix = format!( + "jwt_guest_{}", + bearer(&Claims::valid(), PRIV1, Algorithm::ES256) + ); + + for (label, token) in [ + ("wrong workspace", wrong_ws), + ("wrong key", wrong_key), + ("expired", expired), + ("HS256", hs256), + ("email with an account", has_account), + ("app not in guest mode", not_guest_app), + ("mixed-case account", mixed_case_account), + ("over the 24h lifetime cap", over_lifetime_cap), + ("group-shaped email", group_shaped_email), + ("oversized email", oversized_email), + ("wildcard app_path", wildcard_app_path), + ("oversized token", oversized_token), + ("repeated prefix", repeated_prefix), + ] { + let resp = whoami(port, ws, &token).send().await?; + assert_eq!(resp.status(), 401, "{label} must be refused"); + } + + Ok(()) +} + +/// The workspace switch gates a JWT guest exactly as it gates a signed-in one, at the +/// auth door, so turning guests off closes the JWT entry too. +#[sqlx::test(fixtures("base"))] +async fn guest_jwt_needs_the_workspace_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + let token = bearer(&Claims::valid(), PRIV1, Algorithm::ES256); + + // Switch off (the default): refused. + let resp = whoami(port, ws, &token).send().await?; + assert_eq!( + resp.status(), + 401, + "a JWT guest must be refused while guests are off" + ); + + // Switch on: through. + enable_guests(port, ws, true).await?; + let resp = whoami(port, ws, &token).send().await?; + assert_eq!( + resp.status(), + 200, + "with guests on, the JWT guest is admitted" + ); + + // Off again: closed on the next request. + enable_guests(port, ws, false).await?; + let resp = whoami(port, ws, &token).send().await?; + assert_eq!( + resp.status(), + 401, + "turning guests off closes the JWT guest again" + ); + + Ok(()) +} + +/// A guest JWT is pinned to the workspace its claim names, so it authenticates on no +/// workspace-less route: the arm has no workspace to check the claim against. +#[sqlx::test(fixtures("base"))] +async fn guest_jwt_rejected_on_workspaceless_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", false)).await?; + let token = bearer(&Claims::valid(), PRIV1, Algorithm::ES256); + + let resp = authed( + client().get(format!("http://localhost:{port}/api/users/tokens/list")), + &token, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a guest JWT must not authenticate on a workspace-less route" + ); + + Ok(()) +} + +/// An embed token a JWT guest mints for a sandboxed app is capped at the JWT's own +/// expiry: a JWT has no token row, so the cap is carried through the auth cache. It +/// must not outlive the JWT, which is the guest's only revocation. +#[sqlx::test(fixtures("base"))] +async fn a_guest_jwt_derived_embed_token_is_capped(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = "test-workspace"; + + enable_guests(port, ws, true).await?; + set_guest_jwt_pem(port, ws, PUB1).await?; + create_app(port, ws, app(APP_PATH, "guest", true)).await?; + let secret: String = authed( + client().get(format!( + "http://localhost:{port}/api/w/{ws}/apps/secret_of/{APP_PATH}" + )), + ADMIN_TOKEN, + ) + .send() + .await? + .text() + .await?; + + let claims = Claims::valid(); + let jwt_exp = claims.exp; + let token = bearer(&claims, PRIV1, Algorithm::ES256); + + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/{ws}/apps_u/embed_token/{secret}" + )), + &token, + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let body: serde_json::Value = resp.json().await?; + let child_exp: chrono::DateTime = body["expiration"] + .as_str() + .and_then(|e| e.parse().ok()) + .expect("mint must return the token's expiration"); + assert!( + child_exp.timestamp() as u64 <= jwt_exp, + "the derived embed token ({child_exp}) must not outlive the JWT (exp {jwt_exp})" + ); + + // And it resolves as a guest. + let embed = body["token"].as_str().expect("mint must return a token"); + let resp = whoami(port, ws, embed).send().await?; + assert_eq!(resp.status(), 200); + let me: serde_json::Value = resp.json().await?; + assert_eq!(me["role"], json!("guest")); + + Ok(()) +} + +/// A workspace with no guest key of its own falls back to the instance issuer +/// (`JWT_EXT_JWKS_URL`), so an operator running one issuer configures it once. Verified as a +/// guest here in CE; a full login from that issuer stays EE (`jwt_ext_`). +#[sqlx::test(fixtures("base"))] +async fn no_workspace_key_falls_back_to_the_instance_issuer( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::guest_jwt::{key_source, GuestJwtKeySource}; + let url = "https://issuer.example.com/jwks.json"; + unsafe { std::env::set_var("JWT_EXT_JWKS_URL", url) }; + let src = key_source(&db, "test-workspace").await; + unsafe { std::env::remove_var("JWT_EXT_JWKS_URL") }; + assert!( + matches!(src?, Some(GuestJwtKeySource::JwksUrl(u)) if u == url), + "no workspace key falls back to the instance issuer" + ); + Ok(()) +} diff --git a/backend/tests/app_run_mode_lock_strip.rs b/backend/tests/app_run_mode_lock_strip.rs new file mode 100644 index 0000000000..e2bbf631db --- /dev/null +++ b/backend/tests/app_run_mode_lock_strip.rs @@ -0,0 +1,240 @@ +//! Regression: run mode of `execute_component`'s no-id inline-`raw_code` arm runs +//! *only* the `rawscript/`-pinned `content`, dropping the caller `hash`, +//! `lock`, `modules` and `dedicated_worker` and deriving `path` server-side — +//! all of which would otherwise run or install unpinned code as the app identity. +//! Preview mode keeps honoring the caller's fields. + +use serde_json::json; +use sha2::{Digest, Sha256}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(b: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + b.header("Authorization", format!("Bearer {}", token)) +} + +const CONTENT: &str = "print('benign')\n"; +// A caller lock whose presence is the whole point: if it reaches the job, the +// worker installs it. The value only needs to be recognizable in `v2_job`. +const CALLER_LOCK: &str = "evilpkg @ file:///tmp/attacker-controlled-sdist"; +// A non-codebase-sentinel hash: if it reaches the job as `runnable_id`, the +// worker fetches (and runs) a deployed script by hash instead of the pinned +// content. It need not resolve to a real row — the guard is that it never +// becomes `runnable_id`. +const CALLER_HASH: i64 = 123456789; +// A caller path in someone else's namespace: if it reaches the job as +// `runnable_path` it redirects where the pinned content's relative imports +// resolve. Run mode must instead derive the path from `/`. +const CALLER_PATH: &str = "u/attacker/evil/comp"; + +/// The pin key `execute_component` computes for a no-id inline script: +/// `rawscript/`. +fn rawscript_pin(content: &str) -> String { + let mut h = Sha256::new(); + h.update(content); + format!("rawscript/{:x}", h.finalize()) +} + +fn inline_raw_code(hash: Option, dedicated: bool) -> serde_json::Value { + let mut rc = json!({ + "language": "python3", + "content": CONTENT, + "path": CALLER_PATH, + "lock": CALLER_LOCK, + "modules": { + "m.py": { "content": "print('x')\n", "language": "python3", "lock": CALLER_LOCK } + } + }); + if let Some(h) = hash { + rc["hash"] = json!(h); + } + if dedicated { + rc["dedicated_worker"] = json!(true); + } + rc +} + +/// Fetch `(raw_lock, args-has-_MODULES, runnable_id, tag, runnable_path)` for an +/// enqueued job. +async fn job_fields( + db: &Pool, + uuid: uuid::Uuid, +) -> anyhow::Result<(Option, bool, Option, String, Option)> { + Ok(sqlx::query_as( + "SELECT raw_lock, (args ? '_MODULES'), runnable_id, tag, runnable_path \ + FROM v2_job WHERE id = $1", + ) + .bind(uuid) + .fetch_one(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_strips_caller_lock_and_modules(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip"; + let pin = format!("comp:{}", rawscript_pin(CONTENT)); + + // Deployed Viewer-mode app whose only runnable is an inline script pinned by + // content hash and with no `app_script` row — the legacy `rawscript/` + // case that reaches the no-id run-mode arm this fix touches. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { pin: { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // Run mode (no `force_viewer_static_fields`): the pin authorizes the run, but + // every caller field that selects, installs, or routes code — hash, lock, + // modules, dedicated_worker — must be dropped. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + // The args map is the other injection channel: an inline run is a + // `JobKind::Preview` job, so the worker/executors read `_MODULES` and + // `_TEMP_SCRIPT_REFS` back out of the job args. Both must be stripped. + "args": { + "_MODULES": { "m.py": { "content": "print('evil')\n", "language": "python3" } }, + "_TEMP_SCRIPT_REFS": { "../evil": "deadbeef" } + }, + "component": "comp", + "raw_code": inline_raw_code(Some(CALLER_HASH), true) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 200, + "run-mode pinned inline run must be accepted: {body}" + ); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, runnable_id, tag, runnable_path) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock, None, + "run mode must strip the caller-supplied lock" + ); + assert!( + !has_modules, + "run mode must strip caller modules (both `raw_code.modules` and an `_MODULES` arg)" + ); + let has_temp_refs: bool = + sqlx::query_scalar("SELECT (args ? '_TEMP_SCRIPT_REFS') FROM v2_job WHERE id = $1") + .bind(uuid) + .fetch_one(&db) + .await?; + assert!( + !has_temp_refs, + "run mode must strip a caller `_TEMP_SCRIPT_REFS` arg (relative-import redirect)" + ); + assert_eq!( + runnable_id, None, + "run mode must strip the caller-supplied hash (no substituting a deployed script by hash)" + ); + assert!( + !tag.starts_with("dedi:"), + "run mode must strip caller `dedicated_worker` (no routing to a path-keyed dedicated worker), got tag {tag:?}" + ); + assert_eq!( + runnable_path.as_deref(), + Some(format!("{app_path}/comp").as_str()), + "run mode must derive the path server-side, not trust the caller's (relative-import base)" + ); + + // Preview mode (editor): the caller runs their own code as themselves, so the + // lock and modules are honored — the `/jobs/run/preview`-equivalent path. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "raw_code": inline_raw_code(None, false), + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "preview must be accepted: {body}"); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, _, _, _) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock.as_deref(), + Some(CALLER_LOCK), + "preview must keep the caller-supplied lock" + ); + assert!(has_modules, "preview must keep the caller-supplied modules"); + + Ok(()) +} + +/// A bare `rawscript/` policy key (no `:` prefix, as `empty_triggerables` +/// migrates v1 policies) matches for any `component`, so run mode must not let a +/// path-traversing `component` steer the server-derived `runnable_path`. +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_rejects_traversal_component(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip_bare"; + // Bare key: no `comp:` prefix, so the pin matches regardless of `component`. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { rawscript_pin(CONTENT): { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // A component that isn't a single plain segment steers the derived path's + // base: separators and `..` traverse, and an empty one shifts it up a level. + for bad in ["../../u/attacker/evil", "..", "a/b", ""] { + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": bad, + "raw_code": inline_raw_code(None, false) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "run mode must reject component {bad:?}: got {status}: {body}" + ); + } + + Ok(()) +} diff --git a/backend/tests/app_version_order.rs b/backend/tests/app_version_order.rs new file mode 100644 index 0000000000..e3557ad55c --- /dev/null +++ b/backend/tests/app_version_order.rs @@ -0,0 +1,193 @@ +//! The deployed order of an app's versions is the order they were appended to +//! `app.versions`, not the order of their `created_at`. +//! +//! `app_version.created_at` defaults to `now()`, which in Postgres is the +//! transaction's start time, while the append happens under the app row's lock. +//! Two deploys that overlap therefore land in one order and carry timestamps in +//! the other. The head the editor guards against, and the sequence the diff +//! picker numbers, both have to follow the array. +//! +//! Users from the `base` fixture: test-user (admin, token SECRET_TOKEN). + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const TOKEN: &str = "SECRET_TOKEN"; + +#[sqlx::test(fixtures("base"))] +async fn test_app_head_follows_the_append_order_not_the_timestamps( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let ws = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let res = client + .post(format!("{ws}/apps/create")) + .header("Authorization", format!("Bearer {TOKEN}")) + .json(&json!({ + "path": "u/test-user/order_app", + "summary": "ordered", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert!(res.status().is_success(), "{}", res.text().await?); + + let first: i64 = sqlx::query_scalar( + "SELECT versions[array_upper(versions, 1)] FROM app + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .fetch_one(&db) + .await?; + + // The overlapping deploy: appended after `first`, so it is the version that + // landed, but stamped before it, so a timestamp sort puts it underneath. + let second: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'racer', created_at - interval '1 hour', raw_app + FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + sqlx::query( + "UPDATE app SET versions = array_append(versions, $1::bigint) + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .bind(second) + .execute(&db) + .await?; + + let head: serde_json::Value = client + .get(format!( + "{ws}/apps/get_latest_version/u/test-user/order_app" + )) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + assert_eq!( + head["version"], second, + "the head is the version appended last, not the newest timestamp: {head}" + ); + + let history: Vec = client + .get(format!("{ws}/apps/history/p/u/test-user/order_app")) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + let listed: Vec = history + .iter() + .map(|v| v["version"].as_i64().unwrap()) + .collect(); + assert_eq!( + listed, + vec![second, first], + "the history lists in deployed order, so it leads with the head" + ); + + // Enough versions that a page cannot hold them, so "asked for nothing" and "asked for + // a page" are visibly different answers. + let mut appended = vec![second, first]; + for _ in 0..24 { + let extra: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'bulk', created_at, raw_app FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + sqlx::query( + "UPDATE app SET versions = array_append(versions, $1::bigint) + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .bind(extra) + .execute(&db) + .await?; + appended.insert(0, extra); + } + + let versions_at = |query: &str| { + let url = format!("{ws}/apps/history/p/u/test-user/order_app{query}"); + let client = client.clone(); + async move { + let rows: Vec = client + .get(url) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + Ok::<_, anyhow::Error>( + rows.iter() + .map(|v| v["version"].as_i64().unwrap()) + .collect::>(), + ) + } + }; + + // The deployment-history panel and the CLI read this endpoint without paging, so + // asking for no page has to keep answering with the whole history. + assert_eq!( + versions_at("").await?, + appended, + "an unpaginated request still answers whole" + ); + assert_eq!( + versions_at("?per_page=10").await?, + appended[..10], + "a page holds what was asked for, newest first" + ); + assert_eq!( + versions_at("?per_page=10&page=2").await?, + appended[10..20], + "the next page carries on where the first left off, skipping nothing" + ); + // A page past the end runs off it rather than overflowing into one. (The clamp on an + // asked-for size is pinned where it lives, in `paginate_optional`'s own test.) + assert!( + versions_at("?per_page=10&page=99999999") + .await? + .is_empty(), + "a page past the end is empty" + ); + // Naming only the page still asks for a page, rather than an offset into a listing + // sized to hold everything — which would answer nothing at all. + assert_eq!( + versions_at("?page=1").await?, + appended, + "a first page with no size named holds the history" + ); + + // A version that never entered the deployed sequence — one a restore or a fork copy + // could leave behind — still has to be reachable, after the ones that did. + let stray: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'stray', created_at, raw_app FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + let mut with_stray = appended.clone(); + with_stray.push(stray); + assert_eq!( + versions_at("").await?, + with_stray, + "a version outside app.versions is listed, and after the deployed sequence" + ); + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index fc766253bc..63d5341ba3 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -939,6 +939,127 @@ export function main() { return midValue(); }"#, Ok(()) } +/// A run with local modules and no lock executes the bundle its lock generation built, which +/// kept the imported script's pin; the run must still load the one copy in node_modules, and +/// leave the script's own data alone even where it matches the pinned specifier. +#[sqlx::test(fixtures("base"))] +async fn test_bun_modules_run_loads_imported_pin_from_node_modules( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + insert_deployed_bun_script( + &db, + "f/pinned_import_modules/module", + 41240002, + r#"import * as isNumber from "is-number@6.0.0"; +export const ns = isNumber; +export const label = "is-number@6.0.0";"#, + ) + .await; + + let job = JobPayload::Code(RawCode { + content: r#"import * as isNumber from "is-number"; +import { ns, label } from "/f/pinned_import_modules/module"; +import { local } from "./helper"; +export function main() { return [ns === isNumber, label, local()]; }"# + .into(), + path: Some("f/pinned_import_modules/main".into()), + language: ScriptLang::Bun, + modules: Some(std::collections::HashMap::from([( + "helper.ts".to_string(), + windmill_common::scripts::ScriptModule { + content: "export const local = () => 'local';".into(), + language: ScriptLang::Bun, + lock: None, + }, + )])), + ..RawCode::default() + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + assert_eq!( + result, + serde_json::json!([true, "is-number@6.0.0", "local"]) + ); + Ok(()) +} + +async fn bun_dependency_lock(db: &Pool, port: u16, path: &str, content: &str) -> String { + let deps = RunJob::from(JobPayload::RawScriptDependencies { + script_path: path.into(), + content: content.into(), + language: ScriptLang::Bun, + }) + .run_until_complete(db, false, port) + .await + .json_result() + .unwrap(); + let Some(lock) = deps["lock"].as_str() else { + panic!("the dependency job returned no lock: {deps}"); + }; + lock.to_string() +} + +/// Bundling a locked script resolves a pinned dynamic `import()` as written. Where that fails, both +/// the dependency job and a run that finds no cached bundle must still build it, from the version +/// the lock pins; where bun tolerates the failure, the bundle must stay as written. +#[sqlx::test(fixtures("base"))] +async fn test_bun_bundles_pinned_dynamic_import(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The dependency job saves the script's bundle here; the server binary creates it at startup. + std::fs::create_dir_all(&*windmill_worker::BUN_BUNDLE_CACHE_DIR)?; + + const PATH: &str = "f/pinned_dynamic_import/main"; + // Every `script` call draws its own nonce, so each job below misses every bundle cached before + // it, this test's included, and has to build one: a cached bundle skips the build under test. + // 4.17.20 is not npm's `latest`, so a bundle that lost the pin cannot match by accident. + let script = |body: &str| { + format!( + "export async function main() {{\n {body}\n}}\n// {}", + Uuid::new_v4() + ) + }; + let import = r#"const m = await import("lodash@4.17.20"); return (m.default ?? m).VERSION;"#; + + let lock = bun_dependency_lock(&db, port, PATH, &script(import)).await; + let result = RunJob::from(JobPayload::Code(RawCode { + content: script(import), + path: Some(PATH.into()), + language: ScriptLang::Bun, + lock: Some(lock), + ..RawCode::default() + })) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + assert_eq!(result, serde_json::json!("4.17.20")); + + let tolerated = script(&format!("try {{ {import} }} catch {{ return null; }}")); + let lock = bun_dependency_lock(&db, port, PATH, &tolerated).await; + let (bundle, _) = windmill_worker::compute_bundle_local_and_remote_path( + &tolerated, + &lock, + PATH, + Some(&db), + "test-workspace", + &None, + None, + ) + .await; + assert!(std::fs::read_to_string(bundle)?.contains("lodash@4.17.20")); + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -1088,6 +1209,100 @@ export function main() { Ok(()) } +/// A deployed flow runs an inline step as the `flow_node` its deploy rewrote it into, +/// a `FlowScript` job rather than the preview job the editor runs. A workflow-as-code +/// step's `task()` children must dispatch from that kind too, as re-runs of the same +/// node, or the step passes its editor test and fails once deployed. +/// +/// The step is cached: a child that shared the parent's result-cache key would hand +/// its own result (`10`) back to the parent on resume, in place of the workflow's. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_task_dispatch_from_flow_script(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let node = FlowNodeId(3000000000000011); + let job = RunJob::from(JobPayload::FlowScript { + id: node, + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: Some(60), + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10}) + ); + + let children: Vec<(String, Option, Option)> = sqlx::query_as( + "SELECT kind::text, runnable_id, cache_ttl FROM v2_job WHERE parent_job = $1", + ) + .bind(job.id) + .fetch_all(&db) + .await?; + assert_eq!( + children, + vec![("flowscript".to_string(), Some(node.0), None)], + "the task child re-runs the parent's flow node, outside the result cache" + ); + Ok(()) +} + +/// `task(fn, { cache_ttl })` on an inline task of a deployed flow's step: the child runs +/// the parent's code with the parent's arguments, so its result-cache key carries its +/// step key, or the parent and every sibling would read its result back as their own. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let payload = || JobPayload::FlowScript { + id: FlowNodeId(3000000000000012), + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }; + + let mut children_from_cache = Vec::new(); + for _ in 0..2 { + let job = RunJob::from(payload()) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10, "tripled": 15}) + ); + let from_cache: i64 = sqlx::query_scalar( + "SELECT count(*) FROM job_logs l JOIN v2_job j ON j.id = l.job_id \ + WHERE j.parent_job = $1 AND l.logs LIKE '%found in cache%'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + children_from_cache.push(from_cache); + } + assert_eq!( + children_from_cache, + vec![0, 2], + "the second run serves each task from its own cache entry" + ); + Ok(()) +} + // ============================================================================ // Environment Variable Tests // ============================================================================ diff --git a/backend/tests/datatable_migration_add_down.rs b/backend/tests/datatable_migration_add_down.rs index 3cd39e0fa3..47ea4c253b 100644 --- a/backend/tests/datatable_migration_add_down.rs +++ b/backend/tests/datatable_migration_add_down.rs @@ -142,3 +142,35 @@ async fn test_add_down_to_an_applied_migration(db: Pool) -> anyhow::Re Ok(()) } + +/// A definition saved before `-- role` annotations were parsed can open with a comment that now +/// reads as a malformed one. It never gets through a role check again, so an admin must still be +/// able to finish it rather than be refused on its annotation. +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_add_down_to_a_migration_whose_comment_predates_role_annotations( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + setup_applied_migration_without_down(&db).await?; + let code_up = format!("-- Role table for widgets\n{CODE_UP}"); + sqlx::query("UPDATE datatable_migrations SET code_up = $1 WHERE workspace_id = 'dtmig-ws'") + .bind(&code_up) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = authed(reqwest::Client::new().post(format!( + "http://localhost:{}/api/w/dtmig-ws/workspaces/upsert_datatable_migration/main", + server.addr.port() + ))) + .json(&json!({ + "timestamp": VERSION, + "name": "create_widgets", + "code_up": code_up, + "code_down": CODE_DOWN, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 4520cae848..f4b74525fb 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -521,10 +521,8 @@ def main(): .await .unwrap(); - assert_eq!( - res.text().await.unwrap(), - "app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")" - ); + let deployed: serde_json::Value = res.json().await.unwrap(); + assert_eq!(deployed["path"], "f/rel/root_app_renamed", "{deployed}"); let mut completed = listen_for_completed_jobs(&db).await; in_test_worker(&db, completed.next(), port).await; diff --git a/backend/tests/drafts_move_mirror.rs b/backend/tests/drafts_move_mirror.rs new file mode 100644 index 0000000000..d619381c46 --- /dev/null +++ b/backend/tests/drafts_move_mirror.rs @@ -0,0 +1,74 @@ +//! Moving a draft must carry both of its path keys. +//! +//! A draft value holds a typed path and a mirror the editors keep beside it while +//! it differs from the row's path (`path`/`draft_path`; which is which depends on +//! the kind). The loaders prefer the mirror, so a move that rewrote only the typed +//! key left the mirror naming the old location: reopening the item restored the +//! old path, and the next autosave wrote it back — undoing the move silently. +//! +//! This pins `move_draft`, including that a draft with no mirror never gains one. +//! A deploy's carry follows the keys only where they still name the old path +//! (`drafts_save_follows_move` pins that). + +use serde_json::Value; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn move_to(port: u16, from: &str, to: &str) -> anyhow::Result { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/{from}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&serde_json::json!({ "new_path": to })) + .send() + .await?; + // Asserted here so a rejected move fails as itself, rather than as a JSON + // parse error in the read-back below. + let status = resp.status(); + let body = resp.text().await?; + assert!(status.is_success(), "move {from} -> {to} failed: {body}"); + Ok(body) +} + +/// The stored draft value at `path`, read back through the API so this test needs +/// no `sqlx::query!` (which would want an offline cache entry of its own). +async fn value_at(port: u16, path: &str) -> anyhow::Result { + let body: Value = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/{path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(body + .get("value") + .cloned() + .unwrap_or_else(|| panic!("no draft at {path}: {body}"))) +} + +#[sqlx::test(fixtures("base", "drafts_move_mirror"))] +async fn test_move_carries_both_path_keys(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + move_to(port, "u/test-user/draft_mirror", "u/test-user/renamed").await?; + move_to(port, "u/test-user/draft_plain", "u/test-user/plain2").await?; + + // The mirror follows: left at `u/test-user/friendly` it would win at load and + // walk the item back there. + let moved = value_at(port, "u/test-user/renamed").await?; + assert_eq!(moved["path"], "u/test-user/renamed"); + assert_eq!(moved["draft_path"], "u/test-user/renamed"); + + // A draft that never had a mirror must not be given one. + let plain = value_at(port, "u/test-user/plain2").await?; + assert_eq!(plain["path"], "u/test-user/plain2"); + assert_eq!(plain.get("draft_path"), None, "mirror injected: {plain}"); + + Ok(()) +} diff --git a/backend/tests/drafts_move_taken.rs b/backend/tests/drafts_move_taken.rs new file mode 100644 index 0000000000..40318ca2ef --- /dev/null +++ b/backend/tests/drafts_move_taken.rs @@ -0,0 +1,250 @@ +//! A rename onto a path that already holds a draft is refused. +//! +//! Nothing deployed can sit at a rename's destination (the deploy conflicts on +//! that), but a draft can: a never-deployed item, or a draft left on an archived +//! script. Moving onto it would merge two items or strand a row, so the rename +//! itself fails, in its own transaction, and the source stays deployed. The +//! destination draft here is the deployer's own, which is the same collision. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// Hex form of script hash 7010, the way the API takes a parent hash. +const HEAD_HASH: &str = "0000000000001b62"; + +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_rename_onto_a_draft_is_refused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/mvtaken_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "rename onto a draft was not refused: {body}"); + assert!( + body.contains("already has a draft"), + "unexpected refusal: {body}" + ); + + // The whole deploy rolled back: the source is still the live head, and the + // draft at the destination is untouched. + let head: Value = client + .get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/get/p/u/test-user/mvtaken_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + assert_eq!(head["hash"], HEAD_HASH, "source was replaced: {head}"); + assert_eq!(head["archived"], false, "source was archived: {head}"); + + let draft: Value = client + .get(format!("http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/u/test-user/mvtaken_b")) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + assert_eq!( + draft["value"]["summary"], "B", + "destination draft changed: {draft}" + ); + + Ok(()) +} + +/// A legacy (ownerless) draft occupies its path too: a deploy there deletes it together +/// with the caller's own row, so a move that parks a second draft beside it would discard +/// edits the caller never saw. Only an admin can clear it, so the refusal says so. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_a_legacy_destination(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/mvtaken_b" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/mvtaken_legacy" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "move onto a legacy draft was allowed: {body}"); + assert!( + body.contains("legacy workspace draft") && body.contains("workspace admin"), + "the refusal did not point at the one remedy: {body}" + ); + + // Both rows stayed where they were: the caller's own, and the legacy one the list + // synthesizes under the caller's name. + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + let mut at = list + .iter() + .filter_map(|d| Some((d["kind"].as_str()?, d["path"].as_str()?))) + .filter(|(_, p)| p.starts_with("u/test-user/mvtaken_b") || p.ends_with("mvtaken_legacy")) + .collect::>(); + at.sort(); + assert_eq!( + at, + vec![ + ("script", "u/test-user/mvtaken_b"), + ("script", "u/test-user/mvtaken_legacy") + ], + "{list:?}" + ); + Ok(()) +} + +/// A classic app and a raw app deploy into the same table, so a draft-only move onto +/// the other kind's draft must be refused: deploying either path afterwards deletes +/// the caller's drafts of both kinds, taking the loser's item with it. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_the_other_app_kind(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/raw_app/u/test-user/mvtaken_raw" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/mvtaken_app" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "move onto a classic app draft was allowed: {body}" + ); + assert!( + body.contains("already have a draft at 'u/test-user/mvtaken_app' (app)"), + "the refusal did not name the occupying kind: {body}" + ); + + // Both drafts are untouched. + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + let mut at = list + .iter() + .filter(|d| matches!(d["kind"].as_str(), Some("app") | Some("raw_app"))) + .filter_map(|d| Some((d["kind"].as_str()?, d["path"].as_str()?))) + .collect::>(); + at.sort(); + assert_eq!( + at, + vec![ + ("app", "u/test-user/mvtaken_app"), + ("raw_app", "u/test-user/mvtaken_raw") + ], + "{list:?}" + ); + Ok(()) +} + +/// Teammates' drafts of one item share its path by design, so another user's row is no +/// obstacle — except across the app pair, where the two kinds are different items on one +/// deployed path: deploying either strands the other, and deleting the app takes both. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_another_users_other_app_kind( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let move_to = |kind: &'static str, from: &'static str, to: &'static str| async move { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/{kind}/{from}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": to })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status(), resp.text().await?)) + }; + + let (status, body) = move_to( + "app", + "u/test-user/mvtaken_app", + "u/test-user/mvtaken_theirs", + ) + .await?; + assert_eq!( + status, 400, + "a classic app was moved onto another user's raw app: {body}" + ); + assert!( + body.contains("holds another user's raw app draft"), + "the refusal did not name the occupant: {body}" + ); + + // And the other direction, where the occupant reads as the classic kind. + let (status, body) = move_to( + "raw_app", + "u/test-user/mvtaken_raw", + "u/test-user/mvtaken_app_theirs", + ) + .await?; + assert_eq!( + status, 400, + "a raw app was moved onto another user's classic app: {body}" + ); + assert!( + body.contains("holds another user's app draft"), + "the refusal did not name the occupant: {body}" + ); + + // The same-kind case is the ordinary one: two users' drafts of one raw app. + let (status, body) = move_to( + "raw_app", + "u/test-user/mvtaken_raw", + "u/test-user/mvtaken_theirs", + ) + .await?; + assert!( + status.is_success(), + "a raw app was refused beside another user's raw-app draft: {body}" + ); + Ok(()) +} diff --git a/backend/tests/drafts_nul.rs b/backend/tests/drafts_nul.rs index d7314b8fb6..2a77fe772a 100644 --- a/backend/tests/drafts_nul.rs +++ b/backend/tests/drafts_nul.rs @@ -78,3 +78,46 @@ async fn test_draft_write_strips_nul(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// The lineage the server derives for `draft.base` comes from the same value: read before +/// the sanitizer, a NUL in it costs the draft its base, which then reads as up to date +/// whatever the head is. +#[sqlx::test(fixtures("drafts_nul"))] +async fn test_draft_base_is_derived_from_sanitized_value(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/dnul-ws"); + + let resp = authed(client().post(format!( + "{base}/drafts/update/script/u/dnul-admin/poisoned_base" + ))) + .json(&json!({ + "value": { + "summary": "s", + "path": "u/dnul-admin/poisoned_base", + "content": "x", + "parent_hash": "0000000000001b62\u{0}" + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "save refused: {}", + resp.text().await.unwrap_or_default() + ); + + let stored: Option = sqlx::query_scalar( + "SELECT base FROM draft WHERE workspace_id = 'dnul-ws' AND path = 'u/dnul-admin/poisoned_base'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + stored.as_deref(), + Some("0000000000001b62"), + "the NUL cost the draft its base: {stored:?}" + ); + Ok(()) +} diff --git a/backend/tests/drafts_save_follows_move.rs b/backend/tests/drafts_save_follows_move.rs new file mode 100644 index 0000000000..afbec56c75 --- /dev/null +++ b/backend/tests/drafts_save_follows_move.rs @@ -0,0 +1,587 @@ +//! A draft save addressed to a path its item moved away from lands on the moved draft. +//! +//! A move carries every draft on the item to the new path and records where they +//! went. An editor left open across it still saves to the old path; the server puts +//! the save on the moved draft, keeps the path keys the move gave it, and names the +//! new path so the editor can follow. Without the record the save would plant a +//! phantom draft-only item at the old location. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// Hex form of script hash 7030, the way the API takes a parent hash. +const HEAD_HASH: &str = "0000000000001b76"; + +async fn own_draft_value(port: u16, path: &str) -> anyhow::Result { + let draft: Value = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/{path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(draft["value"].clone()) +} + +async fn own_draft_paths(port: u16) -> anyhow::Result> { + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(list + .iter() + .filter(|d| d["kind"] == "script") + .filter_map(|d| d["path"].as_str().map(String::from)) + .filter(|p| p.starts_with("u/test-user/follow_")) + .collect()) +} + +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_save_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + // Rename from Home: a redeploy of the deployed content at the new path that + // keeps the deployer's own draft, so the draft is carried rather than consumed. + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {}, + "skip_draft_deletion": true + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "rename failed: {}", resp.text().await?); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_b"]); + // A script draft's `path` is where deploying it lands, so it moves with the row. + assert_eq!( + own_draft_value(port, "u/test-user/follow_b").await?["path"], + "u/test-user/follow_b" + ); + + // The editor is still on the old path and writes that path back into the value. + let saved: Value = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { + "path": "u/test-user/follow_a", + "parent_hash": HEAD_HASH, + "summary": "A", + "content": "edited after the move" + } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["status"], "saved", "save refused: {saved}"); + assert_eq!( + saved["path"], "u/test-user/follow_b", + "save did not follow the row: {saved}" + ); + + // The write landed on the carried row; nothing reappeared at the old path. + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_b"]); + let draft = own_draft_value(port, "u/test-user/follow_b").await?; + assert_eq!(draft["content"], "edited after the move", "{draft}"); + assert_eq!(draft["path"], "u/test-user/follow_b", "{draft}"); + + Ok(()) +} + +/// A draft-only move rewrites both path keys. The owner's open editor still carries +/// the typed path it had, which names neither the old nor the new path; the moved +/// draft's own keys have to win, or the save walks the item back. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_save_follows_a_draft_only_move(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/draft_store" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/moved" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + let saved: Value = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/draft_store" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { + "path": "u/test-user/friendly", + "draft_path": "u/test-user/friendly", + "summary": "D", + "content": "edited after the move" + } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["path"], "u/test-user/moved", "{saved}"); + + let draft = own_draft_value(port, "u/test-user/moved").await?; + assert_eq!(draft["content"], "edited after the move", "{draft}"); + assert_eq!(draft["path"], "u/test-user/moved", "{draft}"); + assert_eq!(draft["draft_path"], "u/test-user/moved", "{draft}"); + Ok(()) +} + +/// Rename `from` to `to` the way Home does: redeploy the deployed content at the new +/// path, keeping the deployer's own draft so it is carried rather than consumed. +/// Returns the new head's hash. +async fn rename(port: u16, from_hash: &str, to: &str) -> anyhow::Result { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": to, + "parent_hash": from_hash, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {}, + "skip_draft_deletion": true + })) + .send() + .await?; + let status = resp.status(); + let hash = resp.text().await?; + assert_eq!(status, 201, "rename to {to} failed: {hash}"); + Ok(hash) +} + +/// Save the draft as an editor still bound to `url_path` would. Returns the path the +/// save landed at. +async fn save_at(port: u16, url_path: &str, content: &str) -> anyhow::Result { + let saved: Value = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/{url_path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { "path": url_path, "summary": "A", "content": content, "language": "deno" } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["status"], "saved", "save refused: {saved}"); + Ok(saved["path"].as_str().unwrap_or_default().to_string()) +} + +/// A record is kept to one hop, and a move back to the path it left ends it: both are +/// three statements whose order decides the answer. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_move_records_stay_one_hop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let b = rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + let _c = rename(port, &b, "u/test-user/follow_c").await?; + assert_eq!( + save_at(port, "u/test-user/follow_a", "after two moves").await?, + "u/test-user/follow_c", + "a save at the first path did not reach the last" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_c"]); + Ok(()) +} + +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_move_back_ends_the_record(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let b = rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + let _a = rename(port, &b, "u/test-user/follow_a").await?; + assert_eq!( + save_at(port, "u/test-user/follow_a", "after moving back").await?, + "u/test-user/follow_a", + "a save was routed off the path the item moved back to" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_a"]); + Ok(()) +} + +/// A rename carries a teammate's row too: both its path keys follow, and the version +/// it forked from does not move. A restamp there would clear their out-of-date prompt +/// and let them deploy over the mover's version believing they were current. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_teammates_draft_follows_with_its_base(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + // Read from the pool: the teammate's row is another user's, and this asserts on + // `base`, which no endpoint exposes for someone else's draft. + let row: (String, String, Option) = sqlx::query_as( + "SELECT value::jsonb ->> 'path', value::jsonb ->> 'draft_path', base + FROM draft WHERE workspace_id = 'test-workspace' AND typ = 'script' + AND email = 'test2@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!(row.0, "u/test-user/follow_b", "typed path did not follow"); + assert_eq!(row.1, "u/test-user/follow_b", "mirror did not follow"); + assert_eq!( + row.2.as_deref(), + Some(HEAD_HASH), + "the teammate's base was restamped by someone else's rename" + ); + Ok(()) +} + +/// An item move and then the owner's own move of what is left: the two records have +/// different scopes, so the owner's move has to extend the chain in its own scope or +/// a save addressed to the first path stops at the abandoned middle one. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_an_owner_move_extends_an_item_move(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + // Archiving the script at the new path leaves the carried draft as a draft-only + // item, which its owner can move through `/drafts/move`. + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_b'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_b" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_c" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + assert_eq!( + save_at(port, "u/test-user/follow_a", "after both moves").await?, + "u/test-user/follow_c", + "a save at the first path stopped at the path the owner's move left" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_c"]); + Ok(()) +} + +/// Redeploying at a path an owner's move routed away from ends that route: the live item +/// owns its path again, and its saves must not follow the draft that left. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_redeploy_at_a_routed_path_ends_the_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // `move_draft` ignores archived rows, so an archived script's draft can be moved away. + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + // Unarchiving redeploys at the same path, with the archived version as parent. + rename(port, HEAD_HASH, "u/test-user/follow_a").await?; + + assert_eq!( + save_at(port, "u/test-user/follow_a", "for the live script").await?, + "u/test-user/follow_a", + "a save for the redeployed script followed the draft that moved away" + ); + Ok(()) +} + +/// A draft written before the NUL sanitizer still has to follow a move: its path keys are +/// what a deploy of it would land on, so the carry rewrites them, sanitizing the value it +/// could not otherwise parse. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_poisoned_draft_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The teammate's row, rewritten the way a pre-sanitizer client left one: a real NUL + // escape in the content, both path keys naming the path the item is about to leave. + sqlx::query( + r#"UPDATE draft SET value = '{"path": "u/test-user/follow_a", "draft_path": "u/test-user/follow_a", + "parent_hash": "0000000000001b76", "summary": "A", "content": "a\u0000b"}' + WHERE email = 'test2@windmill.dev'"#, + ) + .execute(&db) + .await?; + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let row: (String, String, String) = sqlx::query_as( + "SELECT value::jsonb ->> 'path', value::jsonb ->> 'draft_path', value::jsonb ->> 'content' + FROM draft WHERE email = 'test2@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!(row.0, "u/test-user/follow_b", "typed path did not follow"); + assert_eq!(row.1, "u/test-user/follow_b", "mirror did not follow"); + assert_eq!(row.2, "ab", "the NUL survived the rewrite"); + Ok(()) +} + +/// A discard whose answer is lost is retried, and the retry finds nothing to delete. It +/// still has to name where the item went: the editor asking is on the path it left, and a +/// reload there lands on nothing. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_retried_discard_still_names_the_destination( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let discard = |token: &'static str| { + let client = client.clone(); + async move { + client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&json!({ "value": null })) + .send() + .await? + .json::() + .await + } + }; + + let first = discard("SECRET_TOKEN").await?; + assert_eq!(first["status"], "saved", "{first}"); + assert_eq!(first["path"], "u/test-user/follow_b", "{first}"); + + let retry = discard("SECRET_TOKEN").await?; + assert_eq!( + retry["status"], "saved", + "the retry was not idempotent: {retry}" + ); + assert_eq!(retry["path"], "u/test-user/follow_b", "{retry}"); + + // A third user has no draft on this item, so their discard deletes nothing and the + // destination is a move record and nothing else: it is answered only to a caller who + // can read that path. + let other = discard("SECRET_TOKEN_3").await?; + assert_eq!(other["status"], "saved", "{other}"); + assert_eq!( + other["path"], + Value::Null, + "a user who cannot read the destination was told where the item went: {other}" + ); + Ok(()) +} + +/// The legacy workspace-level row is carried by a rename like any other draft, and the +/// record that routes saves to it covers every caller — so discarding it from a page that +/// still names the old path has to reach it where it went. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_legacy_discard_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + r#"INSERT INTO draft (workspace_id, path, typ, value, email) + VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "summary": "legacy", "content": "x"}', NULL)"#, + ) + .execute(&db) + .await?; + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "value": null, "legacy": true })) + .send() + .await?; + assert!( + resp.status().is_success(), + "discard failed: {}", + resp.text().await? + ); + + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND email IS NULL", + ) + .fetch_one(&db) + .await?; + assert_eq!( + left, 0, + "the legacy draft survived a discard aimed at its old path" + ); + Ok(()) +} + +/// Deploying a draft that a move carried off an archived script: its parent is still the +/// version at the old path, so the deploy renames from there and carries what is left over +/// — onto the very draft being deployed. That row is this deploy's own, not an item in its +/// way, or the deploy is refused and every retry refuses again. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_moved_draft_deploys_at_its_new_path(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + // Deploy it where it now lives, still parented on the archived version it forked from. + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 2 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 201, "the moved draft could not be deployed: {body}"); + Ok(()) +} + +/// A route is only as good as the item it points at: when an unrelated item claims the +/// destination, a save still addressed to the old path must stay where it is rather than +/// land on that item's draft. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_reused_destination_ends_the_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + // Someone else's item takes the destination, and the moved draft goes with the + // deploy that consumes it. + sqlx::query("DELETE FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/follow_b'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "summary": "unrelated", + "description": "", + "content": "export function main() { return 3 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create failed: {}", resp.text().await?); + + assert_eq!( + save_at(port, "u/test-user/follow_a", "after the destination was reused").await?, + "u/test-user/follow_a", + "a save was routed onto the item that now owns the destination" + ); + Ok(()) +} diff --git a/backend/tests/fixtures/ai_shared_artifacts.sql b/backend/tests/fixtures/ai_shared_artifacts.sql new file mode 100644 index 0000000000..5f745ed0cf --- /dev/null +++ b/backend/tests/fixtures/ai_shared_artifacts.sql @@ -0,0 +1,17 @@ +-- Layers on `base`: a second workspace the superadmin `test-user` is also a member of, so a +-- share can be requested through the wrong workspace's path by a caller the route accepts. + +INSERT INTO workspace (id, name, owner) VALUES + ('test-workspace-2', 'test-workspace-2', 'test-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace-2', 'cloud', 'test-key-2'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('test-workspace-2'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace-2', 'all', 'All users', '{}'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace-2', 'test@windmill.dev', 'test-user', true, 'Admin'); diff --git a/backend/tests/fixtures/drafts_move_mirror.sql b/backend/tests/fixtures/drafts_move_mirror.sql new file mode 100644 index 0000000000..e96d985eae --- /dev/null +++ b/backend/tests/fixtures/drafts_move_mirror.sql @@ -0,0 +1,16 @@ +-- Fixture for the two-path-key move test. +-- +-- Two draft-only scripts owned by test-user, both parked at a generated storage +-- key the way a session-created draft is. One carries the `draft_path` mirror the +-- session editor writes while the typed path differs from that key; the other +-- carries no mirror at all, which is what a plain draft looks like. + +INSERT INTO draft (workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/test-user/draft_mirror', 'script', + '{"path": "u/test-user/friendly", "draft_path": "u/test-user/friendly", + "content": "x", "language": "bun", "summary": "S"}', + 'test@windmill.dev'), + ('test-workspace', 'u/test-user/draft_plain', 'script', + '{"path": "u/test-user/draft_plain", + "content": "x", "language": "bun", "summary": "S"}', + 'test@windmill.dev'); diff --git a/backend/tests/fixtures/drafts_move_taken.sql b/backend/tests/fixtures/drafts_move_taken.sql new file mode 100644 index 0000000000..5fee11b942 --- /dev/null +++ b/backend/tests/fixtures/drafts_move_taken.sql @@ -0,0 +1,37 @@ +-- Fixture for refusing a rename onto a path a draft already occupies. +-- +-- A deployed script at `u/test-user/mvtaken_a` (hash 7010 = 0x1b62), and a +-- never-deployed draft of test-user's own at `u/test-user/mvtaken_b`, the path +-- the rename will target. Nothing deployed lives at the target, so only the +-- draft can refuse the move. + +INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock, extra_perms) +VALUES ('test-workspace', 7010, 'u/test-user/mvtaken_a', + 'export function main() { return 1 }', + 'deno', 'script', 'test-user', '{}', 'A', '', '', '{}'); + +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/mvtaken_b', 'script', + '{"path": "u/test-user/mvtaken_b", "summary": "B", "content": ""}', + 'test@windmill.dev'); + +-- An ownerless (pre per-user drafts) row at a third path: a deploy there deletes it +-- together with the caller's, so it occupies the path just as an own draft does. +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/mvtaken_legacy', 'script', + '{"path": "u/test-user/mvtaken_legacy", "summary": "legacy", "content": ""}', + NULL); + +-- A draft-only classic app and a draft-only raw app of the same owner. They share +-- the `app` table, so one occupies the other's path. The third is another user's raw +-- app: a different item at its own path, not a second copy of anyone's. +INSERT INTO draft (workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/test-user/mvtaken_app', 'app', + '{"summary": "classic", "value": {}}', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_raw', 'raw_app', + '{"summary": "raw", "files": {}}', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_theirs', 'raw_app', + '{"summary": "theirs", "files": {}}', 'test2@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_app_theirs', 'app', + '{"summary": "theirs", "value": {}}', 'test2@windmill.dev'); diff --git a/backend/tests/fixtures/drafts_save_follows_move.sql b/backend/tests/fixtures/drafts_save_follows_move.sql new file mode 100644 index 0000000000..16a41feb1a --- /dev/null +++ b/backend/tests/fixtures/drafts_save_follows_move.sql @@ -0,0 +1,29 @@ +-- Fixture for saving a draft to the path its item moved away from. +-- +-- A deployed script at `u/test-user/follow_a` (hash 7030 = 0x1b76) with the +-- deployer's own draft on it. The test renames the script and then saves the +-- draft from an editor still bound to the old path. + +INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock, extra_perms) +VALUES ('test-workspace', 7030, 'u/test-user/follow_a', + 'export function main() { return 1 }', + 'deno', 'script', 'test-user', '{}', 'A', '', '', '{}'); + +INSERT INTO draft (workspace_id, path, typ, value, email, base) +VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "parent_hash": "0000000000001b76", "summary": "A", "content": "draft"}', + 'test@windmill.dev', '0000000000001b76'); + +-- A draft-only script parked at a generated storage key, its typed path elsewhere. +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/draft_store', 'script', + '{"path": "u/test-user/friendly", "draft_path": "u/test-user/friendly", "summary": "D", "content": "draft"}', + 'test@windmill.dev'); + +-- A teammate's draft on the same deployed script, forked from the same head. The +-- rename must carry it too, without touching the version it forked from. +INSERT INTO draft (workspace_id, path, typ, value, email, base) +VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "draft_path": "u/test-user/follow_a", "parent_hash": "0000000000001b76", "summary": "A", "content": "teammate draft"}', + 'test2@windmill.dev', '0000000000001b76'); diff --git a/backend/tests/fixtures/git_sync_autopull_identity.sql b/backend/tests/fixtures/git_sync_autopull_identity.sql new file mode 100644 index 0000000000..3ae7d03d0e --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_identity.sql @@ -0,0 +1,46 @@ +-- A parent workspace whose auto-pulled repository was last saved by alice, and a fork of +-- it holding only carol, the non-admin who created it. aaron is an admin who sorts before +-- alice; bob is no longer an admin; dora is a workspace admin deactivated on the instance. +-- sam and sue are instance superadmins who are not members: sam's instance username is +-- carol's, sue's is unclaimed. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-parent', 'ap-parent', 'alice@windmill.dev'); +INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ('wm-fork-feat', 'feat', 'carol@windmill.dev', 'ap-parent'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('ap-parent', 'all', 'All users', '{}'), + ('wm-fork-feat', 'all', 'All users', '{}'); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled) VALUES + ('aaron@windmill.dev', 'not-a-real-hash', 'password', false, true, 'aaron', false), + ('alice@windmill.dev', 'not-a-real-hash', 'password', false, true, 'alice', false), + ('bob@windmill.dev', 'not-a-real-hash', 'password', false, true, 'bob', false), + ('carol@windmill.dev', 'not-a-real-hash', 'password', false, true, 'carol', false), + ('dora@windmill.dev', 'not-a-real-hash', 'password', false, true, 'dora', true); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled, username) VALUES + ('sam@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sam', false, 'carol'), + ('sue@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sue', false, 'sue'); + +INSERT INTO usr (workspace_id, email, username, is_admin) VALUES + ('ap-parent', 'aaron@windmill.dev', 'aaron', true), + ('ap-parent', 'alice@windmill.dev', 'alice', true), + ('ap-parent', 'bob@windmill.dev', 'bob', false), + ('ap-parent', 'carol@windmill.dev', 'carol', false), + ('ap-parent', 'dora@windmill.dev', 'dora', true), + ('wm-fork-feat', 'carol@windmill.dev', 'carol', false); + +INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) VALUES + ('ap-parent', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'), + ('wm-fork-feat', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-parent', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false, + "auto_pull":{"enabled":true,"mode":"polling","sync_forks":true, + "enabled_by":"alice@windmill.dev"}}]}'), + ('wm-fork-feat', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false}]}'); diff --git a/backend/tests/fixtures/git_sync_autopull_recovery.sql b/backend/tests/fixtures/git_sync_autopull_recovery.sql new file mode 100644 index 0000000000..6392da50ab --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_recovery.sql @@ -0,0 +1,10 @@ +-- A workspace whose auto-pulled repository last recorded a head-check failure while +-- already synced to head "aaa": the state a recovery write is decided on. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-ws', 'ap-ws', 'test-user'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "auto_pull":{"enabled":true,"mode":"polling", + "last_synced_sha":{"main":"aaa"}, + "last_pull_status":{"success":false,"at":1,"error":"head check failed: x"}}}]}'); diff --git a/backend/tests/fixtures/git_sync_fork_credential.sql b/backend/tests/fixtures/git_sync_fork_credential.sql new file mode 100644 index 0000000000..1c1d8bd1e1 --- /dev/null +++ b/backend/tests/fixtures/git_sync_fork_credential.sql @@ -0,0 +1,55 @@ +-- A parent whose git-sync repository has a recorded credential, and the workspace +-- shapes the credential lookup and the qualification predicate have to tell apart. +-- +-- The credential itself is shared down the fork chain; the recorded *status* is +-- not, because it describes one repository and a fork can repoint its copy of the +-- resource. Forks get a status by fork creation copying it, which no fixture here +-- simulates, so a fork without one is a workspace nothing has checked yet. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('parent-ws', 'parent-ws', 'test-user', NULL), + ('fork-ws', 'fork-ws', 'test-user', 'parent-ws'), + -- A fork of a fork: the shape a fork of a dev workspace takes, and the one a + -- parent-only lookup misses. + ('deep-fork-ws', 'deep-fork-ws', 'test-user', 'fork-ws'), + ('errored-fork-ws', 'errored-fork-ws', 'test-user', 'parent-ws'), + ('orphan-ws', 'orphan-ws', 'test-user', NULL); + +-- A stored credential is encrypted with its own workspace's key. +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('parent-ws', 'cloud', 'parent-key'), + ('fork-ws', 'cloud', 'fork-key'), + ('deep-fork-ws', 'cloud', 'deep-fork-key'), + ('errored-fork-ws', 'cloud', 'errored-fork-key'), + ('orphan-ws', 'cloud', 'orphan-key'); + +-- The parent holds the credential. +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('parent-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":true,"checked_at":1788500000}}]}'), + +-- A fork inherits the repository but not the credential: this is what +-- clone_workspace_data leaves behind. + ('fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- Two levels down, so neither the credential nor its status is one hop away. + ('deep-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- A fork whose own credential has since failed. Its own standing must win over +-- the parent's healthy record rather than being papered over. + ('errored-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":false,"checked_at":1788500000, + "error":"GitLab no longer accepts this token"}}]}'), + +-- No credential and no parent to borrow one from. + ('orphan-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'); + +-- The resource each repository entry names, all pointing at the same repository. +-- The errored fork carries its token in the URL, the way a repository configured +-- by hand does: a plain remote, whatever the chain above it holds. +INSERT INTO resource (workspace_id, path, value, resource_type) VALUES + ('parent-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('deep-fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('errored-fork-ws', 'u/admin/repo', '{"url":"https://oauth2:glpat-inline@gitlab.com/grp/proj.git"}', 'git_repository'), + ('orphan-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'); diff --git a/backend/tests/fixtures/inline_preview_auth.sql b/backend/tests/fixtures/inline_preview_auth.sql index 59fe2dc917..67ca598bda 100644 --- a/backend/tests/fixtures/inline_preview_auth.sql +++ b/backend/tests/fixtures/inline_preview_auth.sql @@ -2,7 +2,9 @@ -- Layered on top of `base` (which provides test-workspace and the non-operator -- `test-user-2`/SECRET_TOKEN_2). Adds an Operator member so we can assert that -- Operators cannot reach the arbitrary-code inline preview path --- (`POST /jobs/run_inline/preview`). +-- (`POST /jobs/run_inline/preview`) with their own token, plus two deployed script +-- jobs of the operator: one running, so we can assert that its WM_TOKEN can, and +-- one queued but not yet pulled, so we can assert that "queued" is not enough. INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) VALUES ('operator@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Operator User'); @@ -12,3 +14,11 @@ INSERT INTO usr(workspace_id, email, username, is_admin, operator, role) VALUES INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('OPERATOR_TOKEN'::bytea), 'hex'), 'OPERATOR_T', 'OPERATOR_TOKEN', 'operator@windmill.dev', 'operator token', false); + +INSERT INTO v2_job(id, workspace_id, kind, runnable_path, created_by, permissioned_as, permissioned_as_email) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'); + +INSERT INTO v2_job_queue(id, workspace_id, scheduled_for, running) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', now(), true), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', now(), false); diff --git a/backend/tests/fixtures/schedule_fork_conflict.sql b/backend/tests/fixtures/schedule_fork_conflict.sql new file mode 100644 index 0000000000..8e81d9ede6 --- /dev/null +++ b/backend/tests/fixtures/schedule_fork_conflict.sql @@ -0,0 +1,46 @@ +-- A three-deep fork chain whose schedule rows were cloned down at fork time. +-- The middle fork has since deleted its copy, so the leaf's schedule shares +-- its cron only with the root — the shape a direct-parent check misses. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('sfc-root', 'sfc-root', 'sfc-admin', NULL), + ('sfc-mid', 'sfc-mid', 'sfc-admin', 'sfc-root'), + ('sfc-leaf', 'sfc-leaf', 'sfc-admin', 'sfc-mid'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('sfc-root', 'cloud', 'sfc-root-key'), + ('sfc-mid', 'cloud', 'sfc-mid-key'), + ('sfc-leaf', 'cloud', 'sfc-leaf-key'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('sfc-root'), ('sfc-mid'), ('sfc-leaf'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('sfc-root', 'all', 'All users', '{}'), + ('sfc-mid', 'all', 'All users', '{}'), + ('sfc-leaf', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('sfc-admin@windmill.dev', 'x', 'password', true, true, 'SFC Admin', 'sfc-admin'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('sfc-root', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-mid', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-leaf', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('SFC_ADMIN_TOKEN'::bytea), 'hex'), 'SFC_ADMIN_', 'SFC_ADMIN_TOKEN', 'sfc-admin@windmill.dev', 't', true); + +-- Enabling pushes the next run, which needs the scheduled script to exist. +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES + ('sfc-leaf', 'sfc-admin', 'export async function main() { return "ok" }', '{}', '', '', 'f/shared/job', 7788001, 'deno', ''); + +INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) +VALUES + ('sfc-root', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', true, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + ('sfc-leaf', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + -- A path only the leaf has: nothing above shares it. + ('sfc-leaf', 'f/shared/own', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'); + +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; diff --git a/backend/tests/fixtures/wac_flow_script.sql b/backend/tests/fixtures/wac_flow_script.sql new file mode 100644 index 0000000000..1b780c6cf4 --- /dev/null +++ b/backend/tests/fixtures/wac_flow_script.sql @@ -0,0 +1,53 @@ +-- A deployed flow whose inline bun step is workflow-as-code calling task(), in the +-- shape the deploy leaves behind: the RawScript module rewritten into a flow_node that +-- the step then runs as a FlowScript job. No lock, so the worker resolves +-- windmill-client at run time like the other bun fixtures. +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', '', '', +'f/system/wac_flow_script', +'{}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"n":{"type":"integer","description":""}},"required":[],"type":"object"}', +'{"modules":[{"id":"a","value":{"type":"flowscript","id":3000000000000011,"language":"bun","input_transforms":{"n":{"expr":"flow_input.n","type":"javascript"}}}}]}', +'system' +); + +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000011, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000011', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}); + +export const main = workflow(async (n: number) => { + const d = await double(n); + return { doubled: d }; +});' +); + +-- The same flow's step with two tasks that cache their own result. +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000012, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000012', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}, { cache_ttl: 60 }); +const triple = task(async (n: number) => { + return n * 3; +}, { cache_ttl: 60 }); + +export const main = workflow(async (n: number) => { + const d = await double(n); + const t = await triple(n); + return { doubled: d, tripled: t }; +});' +); diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 78e45e404b..b6a943a23b 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -3312,3 +3312,60 @@ export function main(i: number) { Ok(()) } + +// A `$flow_expr[...]` step tag is resolved from the flow's state before the step is pushed, and +// one that cannot be resolved fails the step instead of queueing it on a tag no worker serves. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + let step = |id: &str, tag: Option<&str>| { + flow_module( + id, + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: "export function main() { return { lang: 'bun' } }".to_string(), + path: None, + lock: None, + tag: tag.map(str::to_string), + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ) + }; + let flow = FlowValue { + modules: vec![ + step("a", None), + step("b", Some("$flow_expr[results.a.lang]")), + step("c", Some("nobody-serves-$flow_expr[a.lang]")), + ], + same_worker: false, + ..Default::default() + }; + + let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await; + + let b_tag = sqlx::query_scalar::<_, String>( + "SELECT tag FROM v2_job WHERE parent_job = $1 AND flow_step_id = 'b'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + assert_eq!(b_tag, "bun"); + + assert!(!job.success); + let result = job.json_result().unwrap(); + let message = result["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("Could not resolve the step tag `nobody-serves-$flow_expr[a.lang]`"), + "got {result:?}" + ); + + Ok(()) +} diff --git a/backend/tests/folder_default_permissioned_as.rs b/backend/tests/folder_default_permissioned_as.rs index 7aa4f0a003..238d7cdb32 100644 --- a/backend/tests/folder_default_permissioned_as.rs +++ b/backend/tests/folder_default_permissioned_as.rs @@ -630,7 +630,7 @@ async fn test_folder_default_permissioned_as(db: Pool) -> anyhow::Resu ); assert_eq!( policy["on_behalf_of_email"], "group-wm_deployers@windmill.dev", - "app policy.on_behalf_of_email gets folder default email" + "the stored address is derived from that principal" ); // 7b. Admin, non-matching path — acting user diff --git a/backend/tests/git_sync_autopull_identity.rs b/backend/tests/git_sync_autopull_identity.rs new file mode 100644 index 0000000000..4e7562c3f3 --- /dev/null +++ b/backend/tests/git_sync_autopull_identity.rs @@ -0,0 +1,194 @@ +//! An automatic pull runs as the admin stamped on the repository's settings, never as +//! someone picked from the workspace, and stops once that admin is revoked. A fork's +//! pull runs as the parent's pull identity, added to the fork first. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::GitRepositorySettings; +use windmill_git_sync::{reconcile_and_enqueue_pull, reconcile_fork_branch_pull}; + +const PARENT: &str = "ap-parent"; +const FORK: &str = "wm-fork-feat"; +const REPO: &str = "$res:u/alice/repo"; + +fn repo_enabled_by(email: &str) -> GitRepositorySettings { + serde_json::from_value(serde_json::json!({ + "git_repo_resource_path": REPO, + "use_individual_branch": false, + "group_by_folder": false, + "auto_pull": { "enabled": true, "enabled_by": email } + })) + .expect("repository settings") +} + +/// `(created_by, permissioned_as, permissioned_as_email)` of every pull job in `w_id`. +async fn pull_identities( + db: &Pool, + w_id: &str, +) -> anyhow::Result)>> { + Ok(sqlx::query_as( + "SELECT created_by, permissioned_as, permissioned_as_email FROM v2_job \ + WHERE workspace_id = $1 AND kind = 'deploymentcallback'", + ) + .bind(w_id) + .fetch_all(db) + .await?) +} + +fn identity(username: &str) -> (String, String, Option) { + ( + username.to_string(), + format!("u/{username}"), + Some(format!("{username}@windmill.dev")), + ) +} + +async fn recorded_pull_error(db: &Pool, w_id: &str) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(w_id) + .fetch_one(db) + .await?; + Ok( + git_sync["repositories"][0]["auto_pull"]["last_pull_status"]["error"] + .as_str() + .unwrap_or_default() + .to_string(), + ) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_runs_as_the_admin_who_enabled_it(db: Pool) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("alice@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("alice")]); + Ok(()) +} + +/// bob was demoted in the workspace; dora is still a workspace admin but deactivated on +/// the instance. Neither may run the pull, and the failure lands on the status rather +/// than as an error, which a webhook delivery would turn into a failed response. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_fails_on_the_status_once_the_enabling_admin_is_revoked( + db: Pool, +) -> anyhow::Result<()> { + for email in ["bob@windmill.dev", "dora@windmill.dev"] { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by(email), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "{email} must not run the pull"); + let error = recorded_pull_error(&db, PARENT).await?; + assert!(error.contains(email), "{error}"); + } + assert!(pull_identities(&db, PARENT).await?.is_empty()); + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn fork_pull_runs_as_the_parent_admin_added_to_the_fork( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "the fork branch must route to the fork and enqueue" + ); + + let (is_admin, in_all): (bool, bool) = sqlx::query_as( + "SELECT u.is_admin, EXISTS (SELECT 1 FROM usr_to_group g \ + WHERE g.workspace_id = u.workspace_id AND g.usr = u.username AND g.group_ = 'all') \ + FROM usr u WHERE u.workspace_id = $1 AND u.email = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert!( + is_admin && in_all, + "alice must be an admin member of the fork" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("alice")]); + + let grants: i64 = sqlx::query_scalar( + "SELECT count(*) FROM audit_partitioned WHERE workspace_id = $1 \ + AND operation = 'users.git_sync_fork_add' AND resource = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert_eq!(grants, 1, "adding alice to the fork must be audited"); + Ok(()) +} + +/// A superadmin who is not a member runs the pull under their instance username, and +/// `u/` resolves through the workspace's members first. sam's instance username +/// is carol's, so sam's stamp must not run the pull as carol; sue's is unclaimed. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn a_non_member_superadmin_runs_the_pull_only_under_an_unclaimed_username( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sam@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "sam's username belongs to carol"); + assert!(pull_identities(&db, PARENT).await?.is_empty()); + + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sue@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("sue")]); + Ok(()) +} + +/// With no stamp on the parent, the fork pull still runs as the parent's first active +/// admin: the fork holds only its non-admin creator, so no identity resolved in the fork +/// could run it. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn unstamped_fork_pull_runs_as_the_parents_first_admin( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET git_sync = git_sync #- '{repositories,0,auto_pull,enabled_by}' \ + WHERE workspace_id = $1", + ) + .bind(PARENT) + .execute(&db) + .await?; + + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "an unstamped parent must still sync its forks" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("aaron")]); + Ok(()) +} diff --git a/backend/tests/git_sync_autopull_recovery.rs b/backend/tests/git_sync_autopull_recovery.rs new file mode 100644 index 0000000000..d3ba8b7c9f --- /dev/null +++ b/backend/tests/git_sync_autopull_recovery.rs @@ -0,0 +1,143 @@ +//! A recorded auto-pull failure is cleared once the tracked head is observed again +//! at the already-synced sha, and only then: the decision is taken on a snapshot, +//! so the write must re-check the stored row rather than overwrite it. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use std::collections::HashMap; +use uuid::Uuid; +use windmill_common::workspaces::AutoPullStatus; +use windmill_git_sync::{clear_auto_pull_failure, persist_auto_pull_state}; + +const WS: &str = "ap-ws"; +const REPO: &str = "$res:u/admin/repo"; + +/// The failure the fixture records, as the poller would have read it. +fn fixture_failure() -> AutoPullStatus { + AutoPullStatus { + synced_sha: None, + at: 1, + job_id: None, + success: false, + error: Some("head check failed: x".to_string()), + } +} + +fn recovered(head: &str) -> AutoPullStatus { + AutoPullStatus { + synced_sha: Some(head.to_string()), + at: 2, + job_id: None, + success: true, + error: None, + } +} + +async fn stored_auto_pull(db: &Pool) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(WS) + .fetch_one(db) + .await?; + Ok(git_sync["repositories"][0]["auto_pull"].clone()) +} + +/// The recovery every test below runs, decided on the fixture's failure at head +/// "aaa": live in the first test, stale in the two that move the stored state +/// first. +async fn recovery_for_the_fixture_failure(db: &Pool) -> anyhow::Result<()> { + clear_auto_pull_failure( + db, + WS, + REPO, + "main", + "aaa", + &fixture_failure(), + &recovered("aaa"), + ) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn recovery_clears_the_failure_at_the_synced_head(db: Pool) -> anyhow::Result<()> { + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_pull_status"]["success"], true); + assert!(auto_pull["last_pull_status"].get("error").is_none()); + assert_eq!(auto_pull["last_pull_status"]["synced_sha"], "aaa"); + assert_eq!( + auto_pull["last_synced_sha"]["main"], "aaa", + "the sha map is not part of a recovery write" + ); + Ok(()) +} + +/// Between the poller observing head "aaa" unchanged and its recovery write, a +/// webhook may have enqueued newer head "bbb". The stale recovery must leave that +/// optimistic state (sha, success, job id) in place; the job's completion hook +/// relies on it, and rolling the sha back would re-enqueue "bbb" on the next tick. +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn stale_recovery_leaves_a_newer_state_alone(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + let advanced = AutoPullStatus { + synced_sha: Some("bbb".to_string()), + at: 3, + job_id: Some(job_id), + success: true, + error: None, + }; + persist_auto_pull_state( + &db, + WS, + REPO, + &HashMap::from([("main".to_string(), "bbb".to_string())]), + &advanced, + ) + .await?; + + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_synced_sha"]["main"], "bbb"); + assert_eq!(auto_pull["last_pull_status"]["synced_sha"], "bbb"); + assert_eq!(auto_pull["last_pull_status"]["job_id"], job_id.to_string()); + assert_eq!(auto_pull["last_pull_status"]["at"], 3); + Ok(()) +} + +/// The head can stay at "aaa" while a newer failure is recorded (a later head +/// check, a pull job that failed). A recovery decided on the older failure must +/// not paper over the newer one, whether it differs by timestamp or, within the +/// same second, only by its error. +#[sqlx::test(fixtures("git_sync_autopull_recovery"))] +async fn stale_recovery_keeps_a_newer_failure_at_the_same_head( + db: Pool, +) -> anyhow::Result<()> { + let same_sha = HashMap::from([("main".to_string(), "aaa".to_string())]); + for newer in [ + AutoPullStatus { + at: 5, + error: Some("head check failed: later".to_string()), + ..fixture_failure() + }, + AutoPullStatus { + error: Some("head check failed: same second".to_string()), + ..fixture_failure() + }, + ] { + persist_auto_pull_state(&db, WS, REPO, &same_sha, &newer).await?; + + recovery_for_the_fixture_failure(&db).await?; + + let auto_pull = stored_auto_pull(&db).await?; + assert_eq!(auto_pull["last_pull_status"]["success"], false); + assert_eq!(auto_pull["last_pull_status"]["at"], newer.at); + assert_eq!( + auto_pull["last_pull_status"]["error"], + newer.error.as_deref().unwrap() + ); + } + Ok(()) +} diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs new file mode 100644 index 0000000000..9d5eb62dfb --- /dev/null +++ b/backend/tests/git_sync_fork_credential.rs @@ -0,0 +1,376 @@ +//! A fork reaches the git credential held above it in its fork chain. +//! +//! Fork creation copies the parent's git-sync repositories but not the credential, +//! which is stored per workspace so that rotation has one owner. Chains nest (a +//! fork of a dev workspace, a fork of that), so the depth-2 cases here are what +//! keep the lookup from regressing to the parent. +//! +//! The recorded *status* is not shared the same way: it describes one repository, +//! and a fork can repoint its copy of the resource, so each workspace answers from +//! its own record and gets one by fork creation copying it down. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::git_sync_ee::{ + create_repo_webhook, git_app_installations_for, git_credential_for_url, managed_pr_base_branch, + repo_provider, repo_supports_managed_git_features, set_git_credential, GitProvider, +}; +use windmill_common::workspaces::GitCredentialProvider; + +const REPO: &str = "$res:u/admin/repo"; +const URL: &str = "https://gitlab.com/grp/proj.git"; + +/// A repository is managed when a credential is held for the repository its +/// URL names now and the last check found it healthy. The recorded status is +/// keyed by resource path, so alone it would outlive a repoint; the held +/// credential alone says nothing about whether the host still accepts it. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn credential_status_is_a_workspaces_own(db: Pool) -> anyhow::Result<()> { + assert!( + !repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "a healthy status with nothing held behind it does not qualify" + ); + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert!( + repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "the workspace holding both the credential and the recorded status qualifies" + ); + assert!( + !repo_supports_managed_git_features(&db, "fork-ws", REPO).await, + "a fork borrowing the credential with no record of its own does not: the \ + status describes one repository, and this fork's resource could name another" + ); + assert!( + !repo_supports_managed_git_features(&db, "errored-fork-ws", REPO).await, + "a workspace whose own credential failed stays disqualified" + ); + Ok(()) +} + +/// The host a repository talks to is declared when its credential is stored, and +/// travels with the credential down the fork chain. +/// +/// Read from the recorded status instead, a fork answered with the default +/// provider until its own check ran, which is long enough to register a webhook +/// against the wrong receiver. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn the_provider_comes_from_the_credential_and_reaches_forks( + db: Pool, +) -> anyhow::Result<()> { + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitHub, + "with nothing stored there is no declaration to read, so the default stands" + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitLab, + "the workspace that stored it reads its own declaration" + ); + assert_eq!( + repo_provider(&db, "fork-ws", REPO).await, + GitProvider::GitLab, + "and a fork resolving that credential reads it too, without a check of its own" + ); + assert_eq!( + repo_provider(&db, "deep-fork-ws", REPO).await, + GitProvider::GitLab, + "two levels down as well" + ); + assert_eq!( + repo_provider(&db, "orphan-ws", REPO).await, + GitProvider::GitHub, + "a workspace outside the chain resolves no credential and no declaration" + ); + assert_eq!( + repo_provider(&db, "errored-fork-ws", REPO).await, + GitProvider::GitHub, + "a token written into the URL makes the repository a plain remote: the \ + parent's credential is not consulted and no host is declared" + ); + Ok(()) +} + +/// The stored credential is shared with forks and keyed by one repository. +/// +/// Both properties are the point of keeping it in `workspace_settings` under the +/// repository's identity: sharing is what stops a rotation from stranding every +/// fork on a revoked token, and the key is what stops a rewritten resource URL +/// from carrying the token to a host of the writer's choosing. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_fork_reads_an_ancestors_credential_for_the_bound_repository_only( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "the workspace that stored it reads it back" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork stores none of its own and resolves the parent's" + ); + assert_eq!( + git_credential_for_url(&db, "deep-fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork of a fork resolves the root's, two levels up" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", "https://evil.example/grp/proj.git").await?, + None, + "a resource repointed at another repository asks for that one's \ + credential and finds none" + ); + assert_eq!( + git_credential_for_url(&db, "orphan-ws", URL).await?, + None, + "a workspace with no credential and no parent resolves nothing" + ); + Ok(()) +} + +/// One repository's credential is untouched by another's. +/// +/// The key is the repository, so picking a second repository stores beside the +/// first rather than over it. Keyed by the resource instead, a workspace editing +/// one repository's resource to point somewhere else would replace the token the +/// original repository was still syncing with. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn each_repository_keeps_its_own_credential(db: Pool) -> anyhow::Result<()> { + const OTHER_URL: &str = "https://gitlab.com/grp/other.git"; + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-first", + GitCredentialProvider::Gitlab, + ) + .await?; + set_git_credential( + &db, + "parent-ws", + OTHER_URL, + "glpat-second", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-first"), + "storing a second repository's token leaves the first's in place" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second") + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-replacement", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-replacement"), + "storing the same repository again replaces rather than duplicates" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second"), + "and still leaves the other repository alone" + ); + Ok(()) +} + +/// A credential issued for `https` is not served for the `http` spelling. +/// +/// The resource holding the URL is writable by anyone with write on its path, so +/// without the scheme in the key that edit would send the token over cleartext. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_credential_is_not_served_over_a_downgraded_transport( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", "http://gitlab.com/grp/proj.git").await?, + None + ); + Ok(()) +} + +/// A GitLab the server cannot reach is the error reported, not the GitHub App +/// lookup that runs after it: for a self-managed GitLab behind a firewall or an +/// untrusted certificate, "no GitHub App installation" names neither the host +/// nor the cause. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn an_unreachable_gitlab_host_is_the_reported_error( + db: Pool, +) -> anyhow::Result<()> { + let err = create_repo_webhook( + &db, + "parent-ws", + "http://glpat-secret@127.0.0.1:1/grp/proj.git", + "https://windmill.example/api/w/parent-ws/git_sync/webhook/gitlab", + "hook-secret", + ) + .await + .expect_err("nothing listens on port 1"); + assert!( + err.to_string().contains("Could not reach the git host"), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("glpat-secret"), + "the URL credential leaked into the error: {err}" + ); + Ok(()) +} + +/// GitHub App installations are normally copied into a fork, but a workspace +/// attached as a dev workspace, or forked before its parent connected the App, +/// holds none, and neither does anything forked from it. The lookup reaches the +/// nearest workspace up the chain that holds some, and the background App path +/// (PR base resolution here) authenticates with that installation's token. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn app_installations_come_from_the_nearest_ancestor_holding_some( + db: Pool, +) -> anyhow::Result<()> { + use axum::{routing::get, Router}; + use std::sync::{Arc, Mutex}; + + // A stand-in GitHub API: one repository, and a record of who asked for it. + let seen: Arc>> = Arc::new(Mutex::new(vec![])); + let app = Router::new().route( + "/api/v3/repos/acme/repo", + get({ + let seen = seen.clone(); + move |headers: axum::http::HeaderMap| { + let seen = seen.clone(); + async move { + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + seen.lock().unwrap().push(auth); + axum::Json(serde_json::json!({ "default_branch": "trunk" })) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let stub = format!("http://127.0.0.1:{port}"); + + // The root holds the installation, with a cached token so nothing is minted. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'parent-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 42, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "root-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + // The fork's copy of the resource names the App-backed repository. + sqlx::query("UPDATE resource SET value = $1::jsonb WHERE workspace_id = 'deep-fork-ws' AND path = 'u/admin/repo'") + .bind(serde_json::json!({ "url": format!("{stub}/acme/repo.git"), "is_github_app": true })) + .execute(&db) + .await?; + + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?, + ("parent-ws".to_string(), vec![(42, Some(stub.clone()))]), + "two levels down, the root's installations are the ones to use" + ); + assert_eq!( + git_app_installations_for(&db, "orphan-ws").await?, + ("orphan-ws".to_string(), vec![]), + "a workspace with nothing above it resolves nothing" + ); + assert_eq!( + managed_pr_base_branch(&db, "deep-fork-ws", REPO) + .await? + .as_deref(), + Some("trunk"), + "the background App path reaches the repository through the root's installation" + ); + let seen = seen.lock().unwrap().clone(); + assert!( + !seen.is_empty() && seen.iter().all(|auth| auth == "Bearer root-token"), + "every call authenticated with the root's cached token: {seen:?}" + ); + + // A closer holder takes precedence over the root. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'fork-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 7, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "mid-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?.0, + "fork-ws", + "the nearest holder wins over the root" + ); + Ok(()) +} diff --git a/backend/tests/inline_preview_auth.rs b/backend/tests/inline_preview_auth.rs index 97b70fb545..8841719962 100644 --- a/backend/tests/inline_preview_auth.rs +++ b/backend/tests/inline_preview_auth.rs @@ -9,16 +9,33 @@ //! was the incomplete-fix residual of CVE-2026-22683, whose v1.615.0 patch only //! covered the entity-CRUD endpoints and left this direct inline-exec sink open. //! +//! The guard on both routes has one exemption: `wmill.datatable()` called from +//! inside a job the operator is running. Operators can only run deployed code, +//! so a request the job's WM_TOKEN authenticates comes from code a non-operator +//! authored, and the exemption is limited to the request shape the helper sends +//! (PostgreSQL against a `datatable://` database) so a leaked WM_TOKEN cannot +//! be replayed to run anything else. +//! //! This test pins down: -//! - an Operator is rejected by the operator guard (the core fix; pre-fix this -//! reached the inline executor instead of returning 401), and +//! - an Operator's own token is rejected by the operator guard (the core fix; +//! pre-fix this reached the inline executor instead of returning 401), //! - a regular non-operator passes the guard (the fix must not over-block the //! legitimate inline preview flow): in the test harness the worker inline //! utils are not registered, so a caller past the guard gets the distinct -//! "worker inline functions" error rather than the operator rejection. +//! "worker inline functions" error rather than the operator rejection, +//! - an Operator's job token passes the guard for a datatable query while its +//! job is running, on the inline route and on the `/jobs/run/preview` +//! fallback the SDKs use when the worker has no internal server, +//! - the same token is rejected for any other payload (in-process DuckDB, or a +//! `-- database` directive redirecting the query, whether written literally or +//! reached through a `WM_INTERNAL_DB` marker) and for a deferred run, +//! - an Operator's job token for a job that is not running, whether finished or +//! merely queued, is rejected. use serde_json::json; use sqlx::{Pool, Postgres}; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::Authed; use windmill_test_utils::*; fn client() -> reqwest::Client { @@ -38,11 +55,65 @@ fn inline_preview_body() -> serde_json::Value { }) } +/// The request `wmill.datatable("main")` sends: PostgreSQL against `datatable://main`. +fn datatable_query_body() -> serde_json::Value { + json!({ + "language": "postgresql", + "content": "SELECT 1 AS x;", + "args": { "database": "datatable://main" } + }) +} + +/// Mint the WM_TOKEN a job hands its own code: an internally-signed job JWT +/// (note the `job_id` claim) for the fixture's operator, exactly as the worker +/// issues it when the operator runs a deployed script. +async fn operator_job_token(job_id: uuid::Uuid) -> String { + let authed = Authed { + email: "operator@windmill.dev".to_string(), + username: "operator-user".to_string(), + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes: None, + token_prefix: None, + }; + create_jwt_token( + authed, + "test-workspace", + 3600, + Some(job_id), + Some("ephemeral-script".to_string()), + None, + None, + ) + .await + .expect("mint operator job token") +} + const OPERATOR_GUARD_MSG: &str = "Operators cannot run preview jobs"; +/// The fixture's deployed-script jobs of the operator: one running, one queued. +const RUNNING_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000001"; +const QUEUED_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000002"; + +async fn post(url: &str, token: &str, body: &serde_json::Value) -> (u16, String) { + let resp = authed(client().post(url), token) + .json(body) + .send() + .await + .expect("request"); + let status = resp.status().as_u16(); + let body = resp.text().await.expect("body"); + (status, body) +} + #[sqlx::test(fixtures("base", "inline_preview_auth"))] async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; + // The server decodes WM_TOKENs with the same in-process JWT secret, so + // setting it once lets us mint valid ones below. + set_jwt_secret().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -51,12 +122,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // 1. CORE REGRESSION: an Operator must be rejected by the operator guard. // Pre-fix this fell through to the inline executor (arbitrary code // execution); post-fix it returns 401 with the operator guard message. - let resp = authed(client().post(&url), "OPERATOR_TOKEN") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "OPERATOR_TOKEN", &inline_preview_body()).await; assert_eq!( status, 401, "Operator must be rejected from inline preview (got {status}): {body}" @@ -71,12 +137,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // the worker inline utils, so the request proceeds past the guard and // fails later with the distinct "worker inline functions" error — proving // the operator guard did not reject it. - let resp = authed(client().post(&url), "SECRET_TOKEN_2") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "SECRET_TOKEN_2", &inline_preview_body()).await; assert_ne!( status, 401, "non-operator must not be blocked by the operator guard (got {status}): {body}" @@ -86,5 +147,113 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result "non-operator must not hit the operator guard, got: {body}" ); + // 3. The WM_TOKEN of a deployed-script job the Operator is running passes the + // guard for a datatable query: this is `wmill.datatable()` called from + // inside that job. As in 2, the harness then fails with the "worker inline + // functions" error. + let running_job_token = + operator_job_token(uuid::Uuid::parse_str(RUNNING_JOB_ID).unwrap()).await; + let (status, body) = post(&url, &running_job_token, &datatable_query_body()).await; + assert_ne!( + status, 401, + "operator job token of a running job must pass the guard for a datatable query (got {status}): {body}" + ); + assert!( + !body.contains(OPERATOR_GUARD_MSG), + "operator job token of a running job must not hit the operator guard, got: {body}" + ); + + // 4. The same token is rejected for any other payload: the exemption covers + // the datatable request shape only, never in-process DuckDB, and never a + // `-- database` directive, which the executor honors over `args.database`. + let mut redirected = datatable_query_body(); + redirected["content"] = json!("-- database u/test-user/other_db\nSELECT 1 AS x;"); + let mut to_s3 = datatable_query_body(); + to_s3["content"] = json!("-- s3\nSELECT 1 AS x;"); + let mut resource_db = datatable_query_body(); + resource_db["args"]["database"] = json!("$res:u/test-user/other_db"); + // A marker is a single line the directive regexes cannot match; the directive only + // appears once the executor expands it, so the guard must check the expansion. + let mut marker = datatable_query_body(); + marker["content"] = json!(concat!( + r#"-- WM_INTERNAL_DB_SELECT {"table":"t","columnDefs":[{"field":"id","datatype":"int4"}],"#, + r#""whereClause":"true\n-- database u/test-user/other_db\n AND true"}"# + )); + for (label, payload) in [ + ("DuckDB", inline_preview_body()), + ("database directive", redirected), + ("s3 directive", to_s3), + ("resource database", resource_db), + ("marker-expanded database directive", marker), + ] { + let (status, body) = post(&url, &running_job_token, &payload).await; + assert_eq!( + status, 401, + "operator job token must be rejected for a {label} payload (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} payload must be the operator guard, got: {body}" + ); + } + + // 5. An Operator's job token whose job is not running is rejected like the + // operator's own token, whether the job is over (no queue row) or merely + // queued: a WM_TOKEN that leaked through logs cannot be replayed once the + // job is over. + for (label, job_id) in [ + ("finished", uuid::Uuid::new_v4()), + ("queued", uuid::Uuid::parse_str(QUEUED_JOB_ID).unwrap()), + ] { + let token = operator_job_token(job_id).await; + let (status, body) = post(&url, &token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token of a {label} job must be rejected (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} job must be the operator guard, got: {body}" + ); + } + + // 6. The SDKs fall back to `/jobs/run/preview` when the worker has no internal + // server (agent workers). The same exemption applies there: the running + // job's token queues the datatable query (201 with the job id), the + // operator's own token is still refused. + let fallback_url = format!("http://localhost:{port}/api/w/test-workspace/jobs/run/preview"); + let (status, body) = post(&fallback_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 201, + "operator job token of a running job must queue a datatable preview (got {status}): {body}" + ); + let (status, body) = post(&fallback_url, "OPERATOR_TOKEN", &datatable_query_body()).await; + assert_eq!( + status, 401, + "Operator must be rejected from the preview fallback (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection must be the operator guard, got: {body}" + ); + + // 7. A deferred run on the fallback would outlive the running job the + // exemption keys off, so the running job's token cannot schedule one. + for deferral in [ + "scheduled_in_secs=86400", + "scheduled_for=2099-01-01T00:00:00Z", + ] { + let deferred_url = format!("{fallback_url}?{deferral}"); + let (status, body) = post(&deferred_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token must not schedule a deferred preview with {deferral} (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for {deferral} must be the operator guard, got: {body}" + ); + } + Ok(()) } diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index ddd94ea079..5ab870a364 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1442,7 +1442,10 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool) { + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + let mut desired = BTreeMap::new(); + desired.insert( + "base_url".to_string(), + serde_json::json!("https://wm.example.com"), + ); + desired.insert( + "instance_banner".to_string(), + serde_json::json!({ "enabled": true, "message": "down", "link": "javascript:alert(1)" }), + ); + + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err("a javascript: banner link must fail the sync"); + assert!( + err.to_string().contains("instance_banner"), + "the error should name the offending setting, got: {err}" + ); + + assert_eq!( + count_global_settings(&db).await, + before, + "validation must run before anything is applied" + ); + assert!( + get_global_setting(&db, "base_url").await.is_none(), + "the other settings in the same apply must not have been written either" + ); +} + +#[sqlx::test(fixtures("base"))] +async fn declarative_sync_rejects_a_malformed_max_token_expiration(db: Pool) { + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + let mut desired = BTreeMap::new(); + desired.insert( + "max_token_expiration_days".to_string(), + serde_json::json!("7.0"), + ); + + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err("a ceiling the token routes cannot read must fail the sync"); + assert!( + err.to_string().contains("max_token_expiration_days"), + "the error should name the offending setting, got: {err}" + ); + assert_eq!(count_global_settings(&db).await, before); +} + +#[sqlx::test(fixtures("base"))] +async fn declarative_sync_rejects_an_unusable_default_allowed_origins(db: Pool) { + // The declarative writers (the sync-config CLI, the operator's ConfigMap + // sync) do not run the HTTP layer's pre-write hook, so an origin list that + // cannot be parsed would persist here, be dropped at boot, and leave the + // instance with no restriction at all. + clear_settings_and_configs(&db).await; + let before = count_global_settings(&db).await; + + for bad in [ + serde_json::json!([""]), + serde_json::json!(["https://a.example,https://b.example"]), + serde_json::json!("null"), + ] { + let mut desired = BTreeMap::new(); + desired.insert( + "http_route_default_allowed_origins".to_string(), + bad.clone(), + ); + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err(&format!("{bad} must fail the sync")); + assert!( + err.to_string() + .contains("http_route_default_allowed_origins"), + "the error should name the offending setting, got: {err}" + ); + } + + assert_eq!( + count_global_settings(&db).await, + before, + "a rejected sync must not have persisted anything" + ); + + // A usable list still syncs. + let mut desired = BTreeMap::new(); + desired.insert( + "http_route_default_allowed_origins".to_string(), + serde_json::json!(["https://app.example.com"]), + ); + windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect("a valid origin list must sync"); +} diff --git a/backend/tests/job_token_log_masking.rs b/backend/tests/job_token_log_masking.rs new file mode 100644 index 0000000000..5d7de27218 --- /dev/null +++ b/backend/tests/job_token_log_masking.rs @@ -0,0 +1,108 @@ +/* + * The job's own token (`$WM_TOKEN`) stays valid well past the job it was minted + * for, and job logs are persisted to `job_logs` and optionally to object storage, + * so a script that echoes the token would otherwise park a live credential in + * durable storage. `run_worker` registers the token with `sensitive_log_masks` + * for the job it pulled; this pins that the persisted log carries the masked form. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, +}; +use windmill_test_utils::*; + +/// Prefix of a serialized job token: `jwt_` plus the base64 of a JWT header. +/// The masked form keeps only `jwt` + the last three characters, so it never matches. +const RAW_TOKEN_PREFIX: &str = "jwt_ey"; + +#[sqlx::test(fixtures("base"))] +async fn test_job_token_masked_in_persisted_logs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "echo \"running with --token $WM_TOKEN\"".to_string(), + path: None, + lock: None, + language: ScriptLang::Bash, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .run_until_complete(&db, false, port) + .await; + assert!(job.success, "job should have succeeded"); + + let logs = + sqlx::query_scalar::<_, Option>("SELECT logs FROM job_logs WHERE job_id = $1") + .bind(job.id) + .fetch_one(&db) + .await? + .unwrap_or_default(); + + assert!( + !logs.contains(RAW_TOKEN_PREFIX), + "an unmasked job token reached the persisted logs: {logs}" + ); + assert!( + logs.contains("secret value was masked"), + "expected the masking notice in logs: {logs}" + ); + Ok(()) +} + +/// nativets runs V8 in-process and persists `console.log` output through its own +/// channel, so it is masked by a different mechanism than the bash case above and +/// needs its own guard. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_job_token_masked_in_nativets_logs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "export async function main() {\n console.log('running with --token ' + process.env.WM_TOKEN);\n return 'ok';\n}".to_string(), + path: None, + lock: None, + language: ScriptLang::Nativets, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .run_until_complete(&db, false, port) + .await; + assert!(job.success, "job should have succeeded"); + + let logs = + sqlx::query_scalar::<_, Option>("SELECT logs FROM job_logs WHERE job_id = $1") + .bind(job.id) + .fetch_one(&db) + .await? + .unwrap_or_default(); + + assert!( + !logs.contains(RAW_TOKEN_PREFIX), + "an unmasked job token reached the persisted logs: {logs}" + ); + assert!( + logs.contains("secret value was masked"), + "expected the masking notice in logs: {logs}" + ); + Ok(()) +} diff --git a/backend/tests/jobs_cross_site_get.rs b/backend/tests/jobs_cross_site_get.rs new file mode 100644 index 0000000000..20aa09a0a5 --- /dev/null +++ b/backend/tests/jobs_cross_site_get.rs @@ -0,0 +1,138 @@ +//! Regression test for cross-site GET CSRF on the job-run endpoints that can run a Hub script. +//! +//! `run_wait_result/p/{path}` and `run_and_stream/p/{path}` answer GET and, for a `hub/` path, +//! run any public Hub script. The session cookie is `SameSite=Lax`, so a browser attaches it +//! to a cross-site top-level GET navigation, and an argument written `$var:` or +//! `$res:` is resolved as the caller: an attacker page could make a logged-in browser +//! run a generic Hub script and hand it the victim's secrets. CORS hides the response but not +//! the side effect. +//! +//! `CrossSiteGetGuard` refuses such a request. Workspace scripts are deliberately not refused, +//! since they only run code the workspace's own members deployed. A request carrying its own +//! credential is allowed; the one case that regresses is a signed-in user clicking a `?token=` +//! Hub-script link from another site, because `extract_token` gives the cookie precedence and +//! exempting the parameter would let `?token=junk` reinstate the vector. +//! +//! This test pins down: +//! - both endpoints refuse a cross-site cookie GET to a Hub script (the core fix), whether +//! `Sec-Fetch-Site` says so or, with no such header (plain http), a cross-host `Referer`, +//! - a junk `token` query parameter does not buy a pass, +//! - the scope: the same request to a workspace script is not refused, +//! - a Hub-script request with its own credential (bearer, or `?token=` and no cookie), or +//! sent as a POST, gets through. +//! +//! No runnable exists and no Hub is contacted. A request that gets past the guard fails as +//! not-found on a workspace path, and on the non-numeric version in `hub/x/...` for a Hub +//! path, which is rejected while resolving the runnable, before any call to the Hub. + +use reqwest::StatusCode; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const HUB_GETS: [&str; 2] = [ + "run_wait_result/p/hub/x/absent", + "run_and_stream/p/hub/x/absent", +]; +const WORKSPACE_GETS: [&str; 2] = [ + "run_wait_result/p/u/test-user/absent", + "run_and_stream/p/u/test-user/absent", +]; + +async fn send(req: reqwest::RequestBuilder) -> anyhow::Result<(StatusCode, String)> { + let resp = req.send().await?; + let status = resp.status(); + Ok((status, resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn test_cross_site_get_cannot_run_hub_scripts(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/jobs", + server.addr.port() + ); + let client = reqwest::Client::new(); + let cookie_get = |path: &str| { + client + .get(format!("{base}/{path}")) + .header("Cookie", "token=SECRET_TOKEN") + }; + + // ---- CORE REGRESSION: a cross-site cookie GET cannot run a Hub script. + for path in HUB_GETS { + let refused = [ + ( + "Sec-Fetch-Site: cross-site", + cookie_get(path).header("Sec-Fetch-Site", "cross-site"), + ), + // Plain http gets no `Sec-Fetch-*` at all, so `Referer` is the only signal left. + ( + "cross-host Referer with no Sec-Fetch-Site", + cookie_get(path).header("Referer", "http://attacker.example/page"), + ), + // The cookie outranks a `token` query parameter when authenticating. + ( + "junk ?token= next to the cookie", + client + .get(format!("{base}/{path}?token=junk")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ]; + for (name, req) in refused { + let (status, body) = send(req).await?; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "{path} [{name}] must be refused: {body}" + ); + } + } + + // ---- Scope: the same request to a workspace script is not refused. + for path in WORKSPACE_GETS { + let (status, body) = send(cookie_get(path).header("Sec-Fetch-Site", "cross-site")).await?; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{path} is a workspace script and must reach the handler: {body}" + ); + } + + // ---- A Hub-script request that carries its own credential, or is a POST, gets through. + let hub = HUB_GETS[1]; + let allowed = [ + ( + "cross-origin bearer token", + client + .get(format!("{base}/{hub}")) + .header("Authorization", "Bearer SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "cross-origin ?token= with no cookie", + client + .get(format!("{base}/{hub}?token=SECRET_TOKEN")) + .header("Sec-Fetch-Site", "cross-site"), + ), + ( + "POST with the cookie", + client + .post(format!("{base}/{hub}")) + .header("Cookie", "token=SECRET_TOKEN") + .header("Sec-Fetch-Site", "cross-site") + .json(&serde_json::json!({})), + ), + ]; + for (name, req) in allowed { + let (status, body) = send(req).await?; + assert!( + body.contains("Invalid hub script version"), + "{name} must get past the guard to runnable resolution (got {status}): {body}" + ); + } + + Ok(()) +} diff --git a/backend/tests/list_jobs.rs b/backend/tests/list_jobs.rs index 82a9e197cc..29a49c1610 100644 --- a/backend/tests/list_jobs.rs +++ b/backend/tests/list_jobs.rs @@ -1199,3 +1199,63 @@ async fn test_wm_labels_from_result_merged_with_static_labels( Ok(()) } + +/// `tag` lives only on `v2_job`, which count_jobs joins only when `tags` is set. +#[sqlx::test(fixtures("base"))] +async fn test_count_completed_jobs_tags_filter(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + for (ws, tag, status) in [ + ("test-workspace", "deno", "success"), + ("test-workspace", "deno", "failure"), + ("test-workspace", "python3", "success"), + ("other-workspace", "deno", "success"), + ] { + let id = uuid::Uuid::new_v4(); + sqlx::query("INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, $2, $3)") + .bind(id) + .bind(ws) + .bind(tag) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, status, duration_ms) VALUES ($1, $2, $3::job_status, 0)", + ) + .bind(id) + .bind(ws) + .bind(status) + .execute(&db) + .await?; + } + + for (query, expected) in [ + ("", 3), + ("tags=deno", 2), + ("tags=deno&success=true", 1), + ("tags=deno,python3&completed_after_s_ago=3600", 3), + ] { + let response = client + .client() + .get(format!( + "{}/w/test-workspace/jobs/completed/count_jobs?{query}", + client.baseurl() + )) + .send() + .await?; + assert!( + response.status().is_success(), + "{query}: {}", + response.text().await? + ); + assert_eq!(response.json::().await?, expected, "{query}"); + } + + Ok(()) +} diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index 1c6987a480..1e9aef3228 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -12,7 +12,7 @@ mod prewarmed_isolate_tests { use windmill_worker::{build_loader, LoaderMode, BUN_PATH}; fn default_annotation() -> NativeAnnotation { - NativeAnnotation { useragent: None, proxy: None } + NativeAnnotation::default() } /// Bundle a TypeScript script into JS suitable for `PrewarmedIsolate`. diff --git a/backend/tests/object_storage_test_ssrf.rs b/backend/tests/object_storage_test_ssrf.rs new file mode 100644 index 0000000000..aa9eaaf74c --- /dev/null +++ b/backend/tests/object_storage_test_ssrf.rs @@ -0,0 +1,97 @@ +//! `POST /api/settings/test_object_storage_config` runs the probe on the API server and reflects the +//! upstream response, so every non-super-admin must be rejected for private/loopback endpoints and +//! the Filesystem backend on every deployment (`CLOUD_HOSTED` is unset here), while a super admin's +//! Filesystem probe still round-trips. Requires the `parquet` feature, like the route. +#![cfg(feature = "parquet")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use windmill_test_utils::*; + +const SUPER_ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const USER_TOKEN: &str = "SECRET_TOKEN_2"; + +async fn test_object_storage( + url: &str, + token: &str, + body: serde_json::Value, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(url) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn object_storage_test_is_restricted_for_non_super_admins_off_cloud( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/settings/test_object_storage_config", + server.addr.port() + ); + + // A loopback "S3 endpoint" standing in for an internal service: the probe must be rejected + // before the server opens a connection to it. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let internal_port = listener.local_addr()?.port(); + let connected = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let connected = connected.clone(); + async move { + while listener.accept().await.is_ok() { + connected.store(true, Ordering::SeqCst); + } + } + }); + let internal_s3 = json!({ + "type": "S3", + "bucket": "bucket", + "region": "us-east-1", + "access_key": "key", + "secret_key": "secret", + "endpoint": format!("http://127.0.0.1:{internal_port}"), + "allow_http": true, + "path_style": true, + }); + let (status, body) = test_object_storage(&url, USER_TOKEN, internal_s3).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a loopback endpoint (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + assert!( + !connected.load(Ordering::SeqCst), + "the server must not connect to the rejected endpoint" + ); + + let tmp = tempfile::tempdir()?; + let filesystem = json!({ "type": "Filesystem", "root_path": tmp.path().to_str().unwrap() }); + let (status, body) = test_object_storage(&url, USER_TOKEN, filesystem.clone()).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a Filesystem backend (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + + // Super admins keep the unrestricted path. + let (status, body) = test_object_storage(&url, SUPER_ADMIN_TOKEN, filesystem).await?; + assert_eq!( + status, 200, + "super admin must be able to test a Filesystem backend (got {status}): {body}" + ); + Ok(()) +} diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 3a81f31021..1a4040d390 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -627,3 +627,121 @@ async fn test_root_job_span_relocated_to_inbound_trace() { expected_uuid_trace ); } + +// ═══════════════════════════════════════════════════════════════════════ +// RESOURCE ATTRIBUTES (OTEL_RESOURCE_ATTRIBUTES) +// ═══════════════════════════════════════════════════════════════════════ + +fn resource_attrs() -> std::collections::HashMap { + otlp_service_resource( + &windmill_common::utils::Mode::Worker, + "fallback-host", + "dev", + ) + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_merges_env_attributes_without_losing_windmill_identity() { + // These take precedence over the hostname argument and over OTEL_RESOURCE_ATTRIBUTES, + // so clear them or an ambient one fails the assertions below for an unrelated reason. + for var in [ + "OTEL_HOST_NAME", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + ] { + std::env::remove_var(var); + } + std::env::set_var( + "OTEL_RESOURCE_ATTRIBUTES", + "k8s.pod.uid=abc-123,service.name=injected,host.name=injected", + ); + let attrs = resource_attrs(); + std::env::remove_var("OTEL_RESOURCE_ATTRIBUTES"); + + // Attributes the deployment injects reach the exporters. + assert_eq!( + attrs.get("k8s.pod.uid").map(String::as_str), + Some("abc-123") + ); + // OTEL_RESOURCE_ATTRIBUTES is the secondary resource, so Windmill's own values still win. + assert_eq!( + attrs.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + attrs.get("host.name").map(String::as_str), + Some("fallback-host") + ); +} + +#[test] +#[serial_test::serial] +fn test_otlp_resource_dedicated_overrides_win() { + // A deployment sets these per pod, e.g. from Kubernetes downward-API labels. The + // competing service.name must lose: the spec ranks OTEL_SERVICE_NAME above it. + std::env::set_var("OTEL_RESOURCE_ATTRIBUTES", "service.name=should-lose"); + std::env::set_var("OTEL_SERVICE_NAME", "windmill-workers"); + std::env::set_var("OTEL_SERVICE_VERSION", "1.802.0"); + std::env::set_var("OTEL_HOST_NAME", "pod-7"); + let overridden = resource_attrs(); + + // An empty value means unset, which is what the downward API yields for a missing label. + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + ] { + std::env::set_var(var, ""); + } + let empty = resource_attrs(); + for var in [ + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_RESOURCE_ATTRIBUTES", + ] { + std::env::remove_var(var); + } + let unset = resource_attrs(); + + assert_eq!( + overridden.get("service.name").map(String::as_str), + Some("windmill-workers") + ); + assert_eq!( + overridden.get("service.version").map(String::as_str), + Some("1.802.0") + ); + assert_eq!( + overridden.get("host.name").map(String::as_str), + Some("pod-7") + ); + + assert_eq!( + empty.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + empty.get("host.name").map(String::as_str), + Some("fallback-host") + ); + assert_eq!( + empty.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); + + // With nothing set at all — the default deployment — SdkProvidedResourceDetector still + // contributes service.name = "unknown_service". Ours has to overwrite it. + assert_eq!( + unset.get("service.name").map(String::as_str), + Some("windmill-worker") + ); + assert_eq!( + unset.get("service.version").map(String::as_str), + Some(windmill_common::utils::GIT_VERSION) + ); +} diff --git a/backend/tests/postgres_trigger_scope.rs b/backend/tests/postgres_trigger_scope.rs index e8ba36ebb9..1080138eb0 100644 --- a/backend/tests/postgres_trigger_scope.rs +++ b/backend/tests/postgres_trigger_scope.rs @@ -25,6 +25,7 @@ fn scoped_authed(scopes: Vec<&str>) -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } diff --git a/backend/tests/preserve_on_behalf_of.rs b/backend/tests/preserve_on_behalf_of.rs index 2a1f3895ca..0c8c0b76a3 100644 --- a/backend/tests/preserve_on_behalf_of.rs +++ b/backend/tests/preserve_on_behalf_of.rs @@ -405,11 +405,12 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { // 7. App: Admin preserves on_behalf_of // ======================================== + // Principal only, so the stored address can only have come from deriving it. let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") .json(&new_app_with_on_behalf_of( "u/test-user/app_admin_preserve", Some("u/original-user"), - Some("original@windmill.dev"), + None, true, )) .send() @@ -434,10 +435,24 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/original-user"), "Admin should preserve app on_behalf_of" ); + // The address is written through from the principal, never taken from the request, so the + // stored copy can only agree with it. assert_eq!( policy.get("on_behalf_of_email").and_then(|v| v.as_str()), Some("original@windmill.dev"), - "Admin should preserve app on_behalf_of_email" + "the stored address is derived from the principal, not the one the client sent" + ); + let resp = authed( + client().get(format!("{base}/apps/get/p/u/test-user/app_admin_preserve")), + "SECRET_TOKEN", + ) + .send() + .await?; + let returned: serde_json::Value = resp.json().await?; + assert_eq!( + returned["policy"]["on_behalf_of_email"].as_str(), + Some("original@windmill.dev"), + "the response returns the address written through from the principal" ); // ======================================== @@ -476,11 +491,6 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/original-user"), "Deployer should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Deployer should preserve app on_behalf_of_email" - ); // ======================================== // 9. App: Non-admin cannot preserve @@ -518,10 +528,26 @@ async fn test_preserve_on_behalf_of(db: Pool) -> anyhow::Result<()> { Some("u/test-user-2"), "Non-admin should have their own permissioned_as as app on_behalf_of" ); + + // ======================================== + // 9b. App: a policy naming two different accounts is rejected + // ======================================== + + // This is the shape a workspace deploy produces when it carries the source + // workspace's principal beside the target's address. + let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app_with_on_behalf_of( + "u/test-user/app_mismatched_pair", + Some("u/original-user"), + Some("test2@windmill.dev"), + true, + )) + .send() + .await?; assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("test2@windmill.dev"), - "Non-admin should have their own email as app on_behalf_of_email" + resp.status(), + 400, + "a policy whose two halves name different accounts must be rejected" ); // ======================================== @@ -1238,11 +1264,6 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/original-user"), "Admin update should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Admin update should preserve app on_behalf_of_email" - ); // ======================================== // Deployer updates with preserve flag @@ -1304,11 +1325,6 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/original-user"), "Deployer update should preserve app on_behalf_of" ); - assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("original@windmill.dev"), - "Deployer update should preserve app on_behalf_of_email" - ); // ======================================== // Non-admin cannot preserve on update @@ -1370,10 +1386,62 @@ async fn test_app_update_preserves_on_behalf_of(db: Pool) -> anyhow::R Some("u/test-user-2"), "Non-admin update should overwrite app on_behalf_of with their own" ); + + Ok(()) +} + +/// A superadmin acting outside their workspaces has no `usr` row, so the per-workspace rename +/// sweep never reaches the apps that name them. Their principal is their instance username, so +/// without a global sweep a rename leaves those apps naming an account that resolves to nobody. +#[sqlx::test(fixtures("preserve_on_behalf_of"))] +async fn test_rename_sweeps_external_superadmin_app_identity( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + let path = "u/original-user/app_run_by_external_superadmin"; + + let resp = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app_with_on_behalf_of( + path, + Some("u/superadmin-external"), + Some("superadmin-external@windmill.dev"), + true, + )) + .send() + .await?; assert_eq!( - policy.get("on_behalf_of_email").and_then(|v| v.as_str()), - Some("test2@windmill.dev"), - "Non-admin update should overwrite app on_behalf_of_email with their own" + resp.status(), + 201, + "Should create app: {}", + resp.text().await? + ); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/users/rename/superadmin-external@windmill.dev" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "new_username": "superadmin_renamed" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "Should rename: {}", resp.text().await?); + + let app = sqlx::query!( + "SELECT policy FROM app WHERE path = $1 AND workspace_id = $2", + path, + "test-workspace" + ) + .fetch_one(&db) + .await?; + assert_eq!( + app.policy.get("on_behalf_of").and_then(|v| v.as_str()), + Some("u/superadmin_renamed"), + "the rename should follow the principal an app names" ); Ok(()) @@ -2855,10 +2923,8 @@ async fn test_reject_reserved_sentinel_on_behalf_of(db: Pool) -> anyho resp.text().await? ); - // App: a real superadmin on_behalf_of is *allowed* at deploy (deployers may - // deploy on behalf of any real user). The escalation is closed at execution - // by the job-token cap, not by restricting what can be stored, so even a - // superadmin email pinned onto an unrelated principal deploys fine here. + // App: a real superadmin's address pinned onto an unrelated principal is a pair naming two + // accounts, which the principal-authoritative policy refuses. let resp = authed( client().post(format!("{base}/apps/create")), "DEPLOYER_TOKEN", @@ -2873,12 +2939,14 @@ async fn test_reject_reserved_sentinel_on_behalf_of(db: Pool) -> anyho .await?; assert_eq!( resp.status(), - 201, - "a real superadmin on_behalf_of is allowed at deploy (capped at execution): {}", + 400, + "a superadmin address beside an unrelated principal must be refused: {}", resp.text().await? ); - // App: a consistently named real superadmin identity is likewise allowed. + // App: a real superadmin on_behalf_of is allowed at deploy when consistently named (deployers + // may deploy on behalf of any real user); the escalation is closed at execution by the + // job-token cap, not by restricting what can be stored. let resp = authed( client().post(format!("{base}/apps/create")), "DEPLOYER_TOKEN", diff --git a/backend/tests/relock_noop.rs b/backend/tests/relock_noop.rs new file mode 100644 index 0000000000..a480fc57ae --- /dev/null +++ b/backend/tests/relock_noop.rs @@ -0,0 +1,439 @@ +// These pin language-independent behaviour, and Python is the cheapest runtime that still +// generates a real lock out of relative imports, so the whole file needs that feature. +#![cfg(feature = "python")] + +use sqlx::{Pool, Postgres}; +use tokio_stream::StreamExt; +use windmill_api_client::types::NewScript; +use windmill_common::scripts::{deploy_relocked_version, fetch_script_for_update}; +use windmill_test_utils::*; + +const W: &str = "test-workspace"; + +/// Budget for one wait below, counted completions and drain together. Even against a cold cache +/// these settle in a few seconds, so it only bounds a step that is stuck, and it stays under the +/// 60s cap `in_test_worker` puts on the whole body so the panic names what was being waited on +/// rather than surfacing as a worker timeout. +const WAIT_BUDGET: std::time::Duration = std::time::Duration::from_secs(30); + +/// How often the waits below re-check. The drain reads the queue once per completion it waits +/// this long for, so it doubles as the floor on one turn of that loop. +const POLL: std::time::Duration = std::time::Duration::from_millis(20); + +const A: &str = "def main():\n return 'a'\n"; +const A_COMMENTED: &str = "# same dependencies, different content\ndef main():\n return 'a'\n"; +const A_WITH_TINY: &str = "import tiny\n\ndef main():\n return 'a'\n"; +const B: &str = "from f.rel.a import main as a\n\ndef main():\n return 'b' + a()\n"; +const C: &str = "from f.rel.b import main as b\n\ndef main():\n return 'c' + b()\n"; + +fn py_script(path: &str, content: &str, parent_hash: Option) -> NewScript { + NewScript { + draft_only: None, + content: content.into(), + language: windmill_api_client::types::ScriptLang::Python3, + lock: None, + parent_hash, + path: path.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_secs: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + auto_kind: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + modules: None, + } +} + +#[derive(sqlx::FromRow, Debug)] +struct Version { + hash: i64, + archived: bool, + lock: Option, + created_at: chrono::DateTime, +} + +/// Every version of `path`, oldest first. +async fn versions(db: &Pool, path: &str) -> Vec { + sqlx::query_as( + "SELECT hash, archived, lock, created_at FROM script + WHERE workspace_id = $1 AND path = $2 ORDER BY created_at", + ) + .bind(W) + .bind(path) + .fetch_all(db) + .await + .unwrap() +} + +fn live(versions: &[Version]) -> &Version { + versions.iter().rev().find(|v| !v.archived).unwrap() +} + +/// `(path, status, logs)` of every dependency job created after `since`, in completion order. +async fn dependency_jobs_since( + db: &Pool, + since: chrono::DateTime, +) -> Vec<(String, String, String)> { + sqlx::query_as( + "SELECT j.runnable_path, c.status::text, COALESCE(l.logs, '') FROM v2_job_completed c + JOIN v2_job j ON j.id = c.id + LEFT JOIN job_logs l ON l.job_id = c.id + WHERE j.kind = 'dependencies' AND j.created_at > $1 + ORDER BY c.started_at", + ) + .bind(since) + .fetch_all(db) + .await + .unwrap() +} + +async fn wait_for_jobs( + db: &Pool, + completed: &mut (impl futures::Stream + Unpin), + count: usize, +) { + let deadline = tokio::time::Instant::now() + WAIT_BUDGET; + for i in 0..count { + tokio::time::timeout_at(deadline, completed.next()) + .await + .unwrap_or_else(|_| panic!("only {i} of {count} jobs completed")); + } + // Then let anything else that was queued run out, so a job the assertions say must not + // exist would have shown up here. A dependency job queues its fan-out before it completes, + // so an empty queue is a fixpoint rather than a lull. + loop { + while let Ok(Some(_)) = tokio::time::timeout(POLL, completed.next()).await {} + let queued: i64 = sqlx::query_scalar("SELECT count(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap(); + if queued == 0 { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "the queue never emptied" + ); + } +} + +/// A redeploy of an imported script whose dependencies did not move relocks its importer, +/// and that relock must deploy nothing: no new version, and no dependency job for the +/// importer's own importers. A redeploy that does change the dependencies still walks the +/// whole chain with a new version at each step. +#[sqlx::test(fixtures("base"))] +async fn relative_import_relock_deploys_only_when_the_lock_changed( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + // One at a time: each deploy's dependency job records the importer's edges, and an + // importer whose edges are recorded is what a later relock of it can skip on. + for (path, content) in [("f/rel/a", A), ("f/rel/b", B), ("f/rel/c", C)] { + client + .create_script(W, &py_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + } + let b_before = versions(&db, "f/rel/b").await; + let c_before = versions(&db, "f/rel/c").await; + assert_eq!(b_before.len(), 1); + assert_eq!(c_before.len(), 1); + + // Content-only change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!( + paths, + ["f/rel/a", "f/rel/b"], + "the leaf's own job and one no-op relock of its importer, and nothing for c" + ); + assert!( + jobs[1] + .2 + .contains("Lock unchanged: no new version deployed"), + "b's relock should have found its lock unchanged: {}", + jobs[1].2 + ); + let b_after = versions(&db, "f/rel/b").await; + let c_after = versions(&db, "f/rel/c").await; + assert_eq!( + b_after.len(), + 1, + "an unchanged relock must not mint a version" + ); + assert_eq!(live(&b_after).hash, live(&b_before).hash); + assert_eq!(c_after.len(), 1); + assert_eq!(live(&c_after).hash, live(&c_before).hash); + + // A dependency change on the leaf. + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_WITH_TINY, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/c"]); + for path in ["f/rel/b", "f/rel/c"] { + let vs = versions(&db, path).await; + assert_eq!( + vs.len(), + 2, + "{path}: a changed relock deploys a new version" + ); + assert!( + vs[0].archived && !vs[1].archived, + "{path}: parent archived, child live" + ); + assert!(vs[0].created_at < vs[1].created_at, "{path}: lineage order"); + assert!( + vs[1].lock.as_deref().unwrap_or("").contains("tiny"), + "{path}: the new version carries the new lock: {:?}", + vs[1].lock + ); + } + }, + port, + ) + .await; + + Ok(()) +} + +/// A relock that has to wait on its head's row lock, because a deploy of the same path holds +/// it, must find the version that deploy left and requeue itself for it rather than fail. The +/// blocked statement re-checks only the row it selected, which the deploy archived, and comes +/// back empty; the successor is only visible to a fresh read. +#[sqlx::test(fixtures("base"))] +async fn relock_waiting_on_a_deploy_requeues_for_its_successor( + db: Pool, +) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + in_test_worker( + &db, + async { + for (path, content) in [("f/rel/a", A), ("f/rel/b", B)] { + client + .create_script(W, &py_script(path, content, None)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + } + + // A deploy of b that holds its head's row lock for as long as this transaction lives. + let mut deploy = db.begin().await.unwrap(); + let head = fetch_script_for_update("f/rel/b", W, &mut *deploy) + .await + .unwrap() + .unwrap(); + + let since = chrono::Utc::now(); + let a_hash = live(&versions(&db, "f/rel/a").await).hash; + client + .create_script( + W, + &py_script("f/rel/a", A_COMMENTED, Some(format!("{a_hash:016x}"))), + ) + .await + .unwrap(); + + // b's relock skips generation and reaches its commit, where it waits on the lock. + let deadline = std::time::Instant::now() + WAIT_BUDGET; + let mut waiting = false; + while !waiting && std::time::Instant::now() < deadline { + waiting = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE datname = current_database() AND wait_event_type = 'Lock' + AND query LIKE '%FROM script WHERE path = $1%FOR UPDATE%')", + ) + .fetch_one(&db) + .await + .unwrap(); + if !waiting { + tokio::time::sleep(POLL).await; + } + } + assert!(waiting, "b's relock never reached the row lock"); + + // The deploy lands: the head is archived and a successor with its own lock takes + // its place, while the relock is still waiting. + let lock = head.lock.clone().unwrap(); + let successor = + deploy_relocked_version(&mut deploy, head, None, Some(&lock), None, None) + .await + .unwrap(); + deploy.commit().await.unwrap(); + + // a's own job, the relock that waited, and the relock it queued for the successor. + wait_for_jobs(&db, &mut completed, 3).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/a", "f/rel/b", "f/rel/b"], "{jobs:?}"); + assert!( + jobs.iter().all(|(_, status, _)| status == "success"), + "no relock may fail on the wait: {jobs:?}" + ); + assert!( + jobs[1] + .2 + .contains("was deployed while this lock was generated"), + "the waiting relock should have seen the successor: {}", + jobs[1].2 + ); + assert!( + jobs[2] + .2 + .contains("Lock unchanged: no new version deployed"), + "the requeued relock should find the successor's lock current: {}", + jobs[2].2 + ); + let vs = versions(&db, "f/rel/b").await; + assert_eq!( + vs.len(), + 2, + "the deploy's successor and nothing else: {vs:?}" + ); + assert_eq!(live(&vs).hash, successor); + }, + port, + ) + .await; + + Ok(()) +} + +/// A multi-file importer: on a skipped relock each module gets its own last lock back, not the +/// parent script's, so an import's content-only redeploy leaves the importer alone as well. +#[sqlx::test(fixtures("base"))] +async fn multi_file_importer_relock_is_a_no_op_too(db: Pool) -> anyhow::Result<()> { + std::env::set_var("DEPENDENCY_JOB_DEBOUNCE_DELAY", "0"); + let (client, port, _s) = init_client(db.clone()).await; + let mut completed = listen_for_completed_jobs(&db).await; + + let py = |path: &str, content: &str, parent_hash: Option, with_module: bool| { + let mut ns = py_script(path, content, parent_hash); + if with_module { + ns.modules = Some(std::collections::HashMap::from([( + "helper.py".to_string(), + serde_json::json!({ + "content": "def greet(x):\n return 'hi ' + x\n", + "language": "python3" + }), + )])); + } + ns + }; + async fn module_lock(db: &Pool) -> Option { + sqlx::query_scalar( + "SELECT modules->'helper.py'->>'lock' FROM script + WHERE workspace_id = $1 AND path = 'f/rel/pb' AND archived = false", + ) + .bind(W) + .fetch_one(db) + .await + .unwrap() + } + + in_test_worker( + &db, + async { + client + .create_script(W, &py("f/rel/pa", "def main():\n return 'a'\n", None, false)) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + client + .create_script( + W, + &py( + "f/rel/pb", + "from f.rel.pa import main as a\nfrom .helper import greet\n\ndef main():\n return greet(a())\n", + None, + true, + ), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 1).await; + let lock_before = module_lock(&db).await; + assert!(lock_before.is_some(), "the module got a lock of its own on deploy"); + + let since = chrono::Utc::now(); + let pa_hash = live(&versions(&db, "f/rel/pa").await).hash; + client + .create_script( + W, + &py( + "f/rel/pa", + "# same dependencies\ndef main():\n return 'a'\n", + Some(format!("{pa_hash:016x}")), + false, + ), + ) + .await + .unwrap(); + wait_for_jobs(&db, &mut completed, 2).await; + + let jobs = dependency_jobs_since(&db, since).await; + let paths: Vec<&str> = jobs.iter().map(|(p, _, _)| p.as_str()).collect(); + assert_eq!(paths, ["f/rel/pa", "f/rel/pb"], "{jobs:?}"); + assert!( + jobs[1].2.contains("Lock unchanged: no new version deployed"), + "the multi-file importer's relock should be a no-op: {}", + jobs[1].2 + ); + assert_eq!(versions(&db, "f/rel/pb").await.len(), 1); + assert_eq!(module_lock(&db).await, lock_before, "the module keeps its own lock"); + }, + port, + ) + .await; + + Ok(()) +} diff --git a/backend/tests/relock_skip.rs b/backend/tests/relock_skip.rs index 8262a38cf4..bf5cb24cbf 100644 --- a/backend/tests/relock_skip.rs +++ b/backend/tests/relock_skip.rs @@ -266,7 +266,10 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + // Empty content leaves every importer's lock as it was, so only the five direct + // importers of the default deps run a job: an unchanged script relock deploys no + // version and so queues nothing for its own importers. + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; // Note: within a cascade, the same script may be triggered multiple times. // After the first trigger relocks and stores the hash, subsequent triggers skip. @@ -295,7 +298,7 @@ def main(): .await .unwrap(); - in_test_worker(&db, wait_for_jobs_ge(&mut completed, 10), port).await; + in_test_worker(&db, wait_for_jobs_ge(&mut completed, 5), port).await; let skipping_count = count_pattern_in_job_logs(&db, "Skipping relock", before).await; assert!( diff --git a/backend/tests/schedule_fork_conflict.rs b/backend/tests/schedule_fork_conflict.rs new file mode 100644 index 0000000000..9a7ae8ff6d --- /dev/null +++ b/backend/tests/schedule_fork_conflict.rs @@ -0,0 +1,59 @@ +//! Enabling a schedule in a fork warns about every ancestor sharing the path, +//! not only the direct parent: the row was cloned down the whole chain, so the +//! cron is shared with whichever ancestors still hold a copy. + +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn set_enabled( + base: &str, + path: &str, + enabled: bool, + force: bool, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(format!("{base}/api/w/sfc-leaf/schedules/setenabled/{path}")) + .header("Authorization", "Bearer SFC_ADMIN_TOKEN") + .json(&json!({ "enabled": enabled, "force": force })) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("schedule_fork_conflict"))] +async fn enabling_in_a_fork_names_the_nearest_ancestor_sharing_the_path( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}", server.addr.port()); + + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-root"), + "the middle fork deleted its copy, so the root is the one still sharing the cron: {body}" + ); + + let (status, body) = set_enabled(&base, "f/shared/own", true, false).await?; + assert_eq!( + status, 200, + "a path nothing upstream has enables freely: {body}" + ); + + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) + VALUES ('sfc-mid', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin')", + ) + .execute(&db) + .await?; + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-mid"), + "with the parent holding a copy again, it is the nearest and gets named: {body}" + ); + Ok(()) +} diff --git a/backend/tests/script_auto_parent_archived.rs b/backend/tests/script_auto_parent_archived.rs index 5115c516c2..58b926b9c4 100644 --- a/backend/tests/script_auto_parent_archived.rs +++ b/backend/tests/script_auto_parent_archived.rs @@ -9,8 +9,9 @@ //! `lineage must be linear: no 2 scripts can have the same parent` error //! whenever that archived hash already had a child from the prior rename. //! -//! The fix clears `parent_hash` to `None` in that case so the push starts a -//! fresh lineage instead of failing. +//! Resolution only ever adopts an archived version nothing else descends from, +//! so here it finds none and leaves `parent_hash` at `None`, starting a fresh +//! lineage instead of failing. use serde_json::json; use sqlx::{Pool, Postgres}; @@ -146,3 +147,221 @@ async fn test_auto_parent_starts_fresh_lineage_when_all_versions_archived( Ok(()) } + +/// A path whose only versions are archived and childless — what archiving a path +/// leaves behind, and what a sync push that applies a deletion before the matching +/// update sees. Resolving `auto_parent` to no parent there hashes the deploy exactly +/// as the path's first version was hashed, so an unchanged push is rejected as a +/// duplicate of it. +#[sqlx::test(fixtures("base"))] +async fn test_auto_parent_adopts_archived_head_instead_of_colliding( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_archived_head"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let first_hash: i64 = + sqlx::query_scalar("SELECT hash FROM script WHERE path = $1 AND workspace_id = $2") + .bind(path) + .bind("test-workspace") + .fetch_one(&db) + .await?; + + let resp = authed( + client().post(format!("{base}/scripts/archive/p/{path}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "archiving the path should succeed"); + + // Byte-identical to the first deploy: the collision this guards against needs the + // pushed body to hash the same way the original one did. + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&{ + let mut push = body.clone(); + push["auto_parent"] = json!(true); + push + }) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "re-pushing an archived path must not collide with its own first version, \ + got {status}: {response_body}" + ); + + let active: Vec>> = sqlx::query_scalar( + "SELECT parent_hashes FROM script \ + WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(path) + .bind("test-workspace") + .fetch_all(&db) + .await?; + assert_eq!(active.len(), 1, "exactly one active version expected"); + assert_eq!( + active[0].as_deref(), + Some(&[first_hash][..]), + "the revived version should continue the archived lineage" + ); + + Ok(()) +} + +/// The same redeploy from a caller that names no parent at all — the shape a retried +/// `wmill sync push` takes, once the archive it applied has committed and the path has +/// dropped out of the listing it diffs against. +/// +/// The adopted version supplies the lineage and nothing else: a path is reusable by a +/// different script, which must not start life holding grants nobody gave it. +#[sqlx::test(fixtures("base"))] +async fn test_parentless_redeploy_adopts_the_lineage_but_not_the_grants( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_retired_parentless"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let first_hash: i64 = + sqlx::query_scalar("SELECT hash FROM script WHERE path = $1 AND workspace_id = $2") + .bind(path) + .bind("test-workspace") + .fetch_one(&db) + .await?; + + sqlx::query("UPDATE script SET extra_perms = $1 WHERE hash = $2 AND workspace_id = $3") + .bind(json!({ "u/someone_else": true })) + .bind(first_hash) + .bind("test-workspace") + .execute(&db) + .await?; + + let resp = authed( + client().post(format!("{base}/scripts/archive/p/{path}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "archiving the path should succeed"); + + // Byte-identical to the first deploy and naming no parent: hashed as a first deploy it + // lands on the archived row. + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "a parentless redeploy of a retired path must not collide with its own first \ + version, got {status}: {response_body}" + ); + + let live: Vec<(Option>, serde_json::Value)> = sqlx::query_as( + "SELECT parent_hashes, extra_perms FROM script \ + WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(path) + .bind("test-workspace") + .fetch_all(&db) + .await?; + assert_eq!(live.len(), 1, "exactly one live version expected"); + assert_eq!( + live[0].0.as_deref(), + Some(&[first_hash][..]), + "the redeploy should continue the retired lineage" + ); + assert_eq!( + live[0].1, + json!({}), + "an adopted version's grants must not carry over to whatever reuses its path" + ); + + Ok(()) +} + +/// A soft delete keeps the row, and with it the hash the version was deployed under, so a +/// tombstone has to stay adoptable: skip it and the redeploy hashes straight back onto it. +#[sqlx::test(fixtures("base"))] +async fn test_redeploy_chains_past_a_deleted_version(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let path = "u/test-user/script_deleted_version"; + let body = new_script(path, "export async function main() { return 1; }"); + + let resp = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 201); + let deleted_hash = resp.text().await?; + + let resp = authed( + client().post(format!("{base}/scripts/delete/h/{deleted_hash}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "deleting the version should succeed"); + + let resp = authed( + client().post(format!("{base}/scripts/create?skip_if_noop=true")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + let status = resp.status(); + let response_body = resp.text().await?; + assert_eq!( + status, 201, + "redeploying the content of a deleted version must not collide with its \ + tombstone, got {status}: {response_body}" + ); + + Ok(()) +} diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 8755ebfb57..eec9d3b45a 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -159,3 +159,58 @@ export function main(name: string) { assert_eq!(result, json!("hello world")); Ok(()) } + +/// A multi-file script run without a lock is bundled by the lockfile build. A pinned import in +/// a workspace script it imports must be installed at that version and still resolve at run time. +#[sqlx::test(fixtures("base"))] +async fn test_bun_module_imports_pinned_workspace_script(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{}', '', '', 'f/system/pinned_module', 12350, 'bun', '')", + ) + .bind( + r#" +import _ from "lodash@4.17.20"; +export function lodashVersion() { return _.VERSION; } +"#, + ) + .execute(&db) + .await?; + + let mut modules = HashMap::new(); + modules.insert( + "helper.ts".to_string(), + ScriptModule { + content: "export function label(v: string) { return v; }\n".to_string(), + language: ScriptLang::Bun, + lock: None, + }, + ); + + let job = JobPayload::Code(RawCode { + content: r#" +import { lodashVersion } from "/f/system/pinned_module"; +import { label } from "./helper.ts"; +export function main() { return label(lodashVersion()); } +"# + .to_owned(), + path: Some("f/system/my_script".to_string()), + language: ScriptLang::Bun, + modules: Some(modules), + tag: None, + ..RawCode::default() + }); + + let result = RunJob::from(job) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("4.17.20")); + Ok(()) +} diff --git a/backend/tests/session_workspace_status.rs b/backend/tests/session_workspace_status.rs index b4fc98da5e..2ed4ac503a 100644 --- a/backend/tests/session_workspace_status.rs +++ b/backend/tests/session_workspace_status.rs @@ -3,18 +3,23 @@ //! extractor actually grants. Membership is not the only path: a superadmin is authed into //! any existing workspace without a `usr` row, and `admins` has no `usr` rows at all, so //! answering from `usr` alone reports live workspaces as unresolvable and the client deletes -//! sessions that still work. +//! sessions that still work. `POST /workspaces/session_workspace_retention`, the AI session +//! retention the same client deletes its own copies by, is a workspace setting and answers to +//! the stricter bar, which is why the two are separate routes and tested together. use serde_json::json; use sqlx::{Pool, Postgres}; use std::collections::HashMap; use windmill_test_utils::*; -async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { +async fn post( + port: u16, + route: &str, + token: &str, + ids: &[&str], +) -> anyhow::Result { let resp = reqwest::Client::new() - .post(format!( - "http://localhost:{port}/api/workspaces/session_workspace_status" - )) + .post(format!("http://localhost:{port}/api/workspaces/{route}")) .header("Authorization", format!("Bearer {token}")) .json(&json!({ "workspace_ids": ids })) .send() @@ -23,6 +28,14 @@ async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result anyhow::Result> { + post(port, "session_workspace_status", token, ids).await +} + +async fn retention(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { + post(port, "session_workspace_retention", token, ids).await +} + #[sqlx::test(fixtures("base", "session_workspace_status"))] async fn test_superadmin_reaches_workspaces_without_a_usr_row( db: Pool, @@ -60,3 +73,48 @@ async fn test_superadmin_reaches_workspaces_without_a_usr_row( Ok(()) } + +/// The retention a browser deletes its own copies by is a workspace setting, so unlike the +/// status it is told only to a caller the authed extractor would let in. +#[sqlx::test(fixtures("base", "session_workspace_status"))] +async fn test_session_retention_is_told_only_to_members_who_can_be_authed( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let ids = ["foreign-workspace", "test-workspace", "no-such-workspace"]; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 7}' \ + WHERE workspace_id IN ('test-workspace', 'foreign-workspace')", + ) + .execute(&db) + .await?; + + // test@windmill.dev is a superadmin: authed into every workspace that exists. + let sa = retention(port, "SECRET_TOKEN", &ids).await?; + assert_eq!(sa["test-workspace"], 7); + assert_eq!(sa["foreign-workspace"], 7); + assert!(!sa.contains_key("no-such-workspace")); + + // test2@windmill.dev is a member of test-workspace only. + let usr = retention(port, "SECRET_TOKEN_2", &ids).await?; + assert_eq!(usr["test-workspace"], 7); + assert!(!usr.contains_key("foreign-workspace")); + + // A disabled membership still reconciles its sessions — the status stays `active` — but + // cannot be authed into the workspace, so it is told no setting. + sqlx::query("UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + assert_eq!( + status(port, "SECRET_TOKEN_2", &ids).await?["test-workspace"], + "active" + ); + assert!(!retention(port, "SECRET_TOKEN_2", &ids) + .await? + .contains_key("test-workspace")); + + Ok(()) +} diff --git a/backend/tests/trigger_listener_queries.rs b/backend/tests/trigger_listener_queries.rs index 1463138167..0af99136e7 100644 --- a/backend/tests/trigger_listener_queries.rs +++ b/backend/tests/trigger_listener_queries.rs @@ -178,6 +178,7 @@ fn make_authed() -> windmill_api_auth::ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } diff --git a/backend/tests/v2_job_delete_orphans.rs b/backend/tests/v2_job_delete_orphans.rs index 95cd1673b6..2b11deab77 100644 --- a/backend/tests/v2_job_delete_orphans.rs +++ b/backend/tests/v2_job_delete_orphans.rs @@ -37,7 +37,7 @@ async fn seed_side_rows(db: &Pool, ws: &str, job_id: Uuid) -> anyhow:: .bind(ws) .execute(db) .await?; - // created_seq is assigned by a trigger; inserting a value is rejected. + // created_seq is an identity column; supplying a value is rejected. sqlx::query( "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) VALUES ($1, 'assistant', 'hi', $2)", @@ -121,6 +121,186 @@ async fn test_delete_jobs_removes_side_rows(db: Pool) -> anyhow::Resul Ok(()) } +/// (conversation rows, agent-memory rows) for one conversation. +async fn conversation_and_memory_counts( + db: &Pool, + conversation_id: Uuid, +) -> anyhow::Result<(i64, i64)> { + Ok(( + count( + db, + "SELECT count(*) FROM flow_conversation WHERE id = $1", + conversation_id, + ) + .await?, + count( + db, + "SELECT count(*) FROM ai_agent_memory WHERE conversation_id = $1", + conversation_id, + ) + .await?, + )) +} + +/// A conversation outlives the jobs behind its messages until the last one goes: only then +/// are the row and the agent's memory for it left with nothing, and only then are they +/// deleted. Both halves matter — the surviving half is what a single data-modifying CTE +/// would break, since its emptiness check would read the snapshot from before the delete. +#[sqlx::test(fixtures("base"))] +async fn test_delete_jobs_removes_a_conversation_once_its_last_message_goes( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let first_job = Uuid::new_v4(); + let second_job = Uuid::new_v4(); + insert_job(&db, WS, first_job).await?; + insert_job(&db, WS, second_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + for job_id in [first_job, second_job] { + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'assistant', 'hi', $2)", + ) + .bind(conv_id) + .bind(job_id) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages) + VALUES ($1, $2, 'a', '[]'::jsonb)", + ) + .bind(WS) + .bind(conv_id) + .execute(&db) + .await?; + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[first_job]).await?; + drop(conn); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (1, 1), + "a conversation with a message left must survive, memory included" + ); + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[second_job]).await?; + drop(conn); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (0, 0), + "the last message going should take the conversation and its memory" + ); + Ok(()) +} + +/// Turns that start while retention is collecting their conversation must land, not fail: +/// the conversation lookup locks the row, so each turn waits for the collector's commit, +/// finds the conversation gone, and creates it again — the first insert wins and the other +/// reads its row. Without the lock a turn's message insert is what waits, on the parent +/// row's key lock, and fails its FK check afterwards. +#[sqlx::test(fixtures("base"))] +async fn test_new_turns_wait_for_conversation_cleanup_and_recreate( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let old_job = Uuid::new_v4(); + let new_jobs = [Uuid::new_v4(), Uuid::new_v4()]; + insert_job(&db, WS, old_job).await?; + for job in new_jobs { + insert_job(&db, WS, job).await?; + } + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'user', 'hi', $2)", + ) + .bind(conv_id) + .bind(old_job) + .execute(&db) + .await?; + + // The collector holds the conversation row locked and deleted, uncommitted. + let mut cleanup = db.begin().await?; + windmill_common::jobs::delete_jobs(&mut *cleanup, &[old_job]).await?; + + let turns: Vec<_> = new_jobs + .into_iter() + .map(|new_job| { + let db = db.clone(); + tokio::spawn(async move { + let mut tx = db.begin().await?; + windmill_common::flow_conversations::get_or_create_conversation_with_id( + &mut tx, + WS, + "f/flow", + "test-user", + "hi again", + conv_id, + false, + ) + .await?; + windmill_common::flow_conversations::add_message_to_conversation_tx( + &mut tx, + conv_id, + Some(new_job), + "hi again", + windmill_common::flow_conversations::MessageType::User, + None, + true, + None, + ) + .await?; + tx.commit().await?; + anyhow::Ok(()) + }) + }) + .collect(); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + cleanup.commit().await?; + for turn in turns { + turn.await??; + } + + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?.0, + 1, + "the turns must have created the conversation again, once" + ); + assert_eq!( + count( + &db, + "SELECT count(*) FROM flow_conversation_message WHERE conversation_id = $1", + conv_id, + ) + .await?, + 2, + "both turns' messages should be there" + ); + Ok(()) +} + #[sqlx::test(fixtures("base"))] async fn test_clear_schedule_removes_side_rows(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -192,6 +372,74 @@ async fn test_workspace_delete_removes_side_rows(db: Pool) -> anyhow:: Ok(()) } +/// The purge endpoint carries its own copy of the emptied-conversation rule, so it gets the +/// same guard: the conversation and its memory go with the last message, and not before. +#[sqlx::test(fixtures("base"))] +async fn test_jobs_export_delete_removes_a_conversation_once_its_last_message_goes( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let first_job = Uuid::new_v4(); + let second_job = Uuid::new_v4(); + insert_job(&db, WS, first_job).await?; + insert_job(&db, WS, second_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + for job_id in [first_job, second_job] { + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'assistant', 'hi', $2)", + ) + .bind(conv_id) + .bind(job_id) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages) + VALUES ($1, $2, 'a', '[]'::jsonb)", + ) + .bind(WS) + .bind(conv_id) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let purge = |job_id: Uuid| async move { + reqwest::Client::new() + .post(format!("http://localhost:{port}/api/w/{WS}/jobs/delete")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&[job_id]) + .send() + .await + }; + + assert!(purge(first_job).await?.status().is_success()); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (1, 1), + "a conversation with a message left must survive the purge endpoint too" + ); + + assert!(purge(second_job).await?.status().is_success()); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (0, 0), + "the last message going should take the conversation and its memory" + ); + Ok(()) +} + /// The `/jobs/delete` purge endpoint must scope every side-table delete to the path /// workspace. A `test-workspace` admin passing a job id from another workspace must not be /// able to delete that workspace's job or side rows (the side tables no longer cascade, so diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs index 8ecf6e98b6..e93979e9c7 100644 --- a/backend/tests/volume_tests.rs +++ b/backend/tests/volume_tests.rs @@ -537,6 +537,64 @@ fn test_asset_kind_volume_variant() { #[cfg(feature = "parquet")] #[sqlx::test(fixtures("base"))] async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + let code = r#"// volume: test-vol data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("data/hello.txt", "utf-8"); + writeFileSync("data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("data/output.txt"), + }; +}"#; + run_volume_with_default_stack(db, ScriptLang::Bun, code).await +} + +#[cfg(all(feature = "parquet", feature = "private", feature = "php"))] +#[sqlx::test(fixtures("base"))] +async fn test_php_volume_with_default_stack(db: Pool) -> anyhow::Result<()> { + // Nested calls exercise the parser's fallback chain; flattening them weakens this guard. + let code = r#" $content, + "output_exists" => in_array("output.txt", array_values(array_diff(scandir("data"), ['.', '..']))), + ]; +}"#; + + run_volume_with_default_stack(db, ScriptLang::Php, code).await +} + +#[cfg(feature = "parquet")] +async fn run_volume_with_default_stack( + db: Pool, + language: ScriptLang, + code: &'static str, +) -> anyhow::Result<()> { + // CI raises RUST_MIN_STACK; keep the worker at Tokio's default to catch regressions. + tokio::task::spawn_blocking(move || { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .thread_stack_size(2 * 1024 * 1024) + .enable_all() + .build()? + .block_on(run_volume_sql_worker_e2e(db, language, code)) + }) + .await? +} + +#[cfg(feature = "parquet")] +async fn run_volume_sql_worker_e2e( + db: Pool, + language: ScriptLang, + code: &str, +) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -571,24 +629,11 @@ async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; // 3. Push the job and run with SQL-connected worker - let code = r#"// volume: test-vol data - -import { readFileSync, writeFileSync, existsSync } from "fs"; - -export function main() { - const content = readFileSync("data/hello.txt", "utf-8"); - writeFileSync("data/output.txt", "written by sql worker"); - return { - read_content: content, - output_exists: existsSync("data/output.txt"), - }; -}"#; - let job = JobPayload::Code(RawCode { hash: None, content: code.to_string(), path: None, - language: ScriptLang::Bun, + language, lock: None, cache_ttl: None, cache_ignore_s3_path: None, diff --git a/backend/tests/wac_child_completion_wakes_parent.rs b/backend/tests/wac_child_completion_wakes_parent.rs new file mode 100644 index 0000000000..23c32573ab --- /dev/null +++ b/backend/tests/wac_child_completion_wakes_parent.rs @@ -0,0 +1,189 @@ +//! A WAC v2 parent parks on its dispatched children and is woken by their +//! completions. A child does not always complete through the worker that ran it: +//! the zombie monitor and a force cancel both go straight to +//! `add_completed_job_error`. The parent must be woken from there too, or it sits +//! out its whole suspend window and then runs the task a second time. + +use serde_json::{json, Value}; +use sqlx::{types::Json, Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::{add_completed_job, add_completed_job_error, get_mini_completed_job}; + +const W_ID: &str = "test-workspace"; + +async fn insert_job(db: &Pool, id: Uuid, parent: Option) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner, parent_job) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', \ + 'script', 'bun', 'u/test-user/wac', 'bun', true, $3)", + ) + .bind(id) + .bind(W_ID) + .bind(parent) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag) \ + VALUES ($1, $2, now(), true, 'bun')", + ) + .bind(id) + .bind(W_ID) + .execute(db) + .await?; + Ok(()) +} + +/// A parent parked on `steps` (step key → child job), the shape +/// `handle_wac_v2_output` leaves behind once the children are pushed. +async fn plant_parked_parent(db: &Pool, steps: &[(&str, Uuid)]) -> anyhow::Result { + let parent = Uuid::new_v4(); + insert_job(db, parent, None).await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 days' \ + WHERE id = $1", + ) + .bind(parent) + .bind(steps.len() as i32) + .execute(db) + .await?; + let job_ids: serde_json::Map = steps + .iter() + .map(|(k, id)| (k.to_string(), json!(id.to_string()))) + .collect(); + let keys: Vec<&str> = steps.iter().map(|(k, _)| *k).collect(); + sqlx::query("INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, $2)") + .bind(parent) + .bind(json!({ + "_checkpoint": { + "completed_steps": {}, + "pending_steps": { "mode": "dispatch", "keys": keys, "job_ids": job_ids }, + "job_ids": job_ids, + } + })) + .execute(db) + .await?; + for (_, child) in steps { + insert_job(db, *child, Some(parent)).await?; + } + Ok(parent) +} + +async fn parent_state(db: &Pool, parent: Uuid) -> anyhow::Result<(i32, bool, Value)> { + let (suspend, parked, status): (i32, bool, Value) = sqlx::query_as( + "SELECT q.suspend, q.suspend_until IS NOT NULL, s.workflow_as_code_status \ + FROM v2_job_queue q JOIN v2_job_status s USING (id) WHERE q.id = $1", + ) + .bind(parent) + .fetch_one(db) + .await?; + Ok((suspend, parked, status)) +} + +/// The zombie monitor's path: `handle_job_error` → `add_completed_job_error`, never +/// the worker's result processor. The parent must come out of it pullable, with the +/// failure recorded under the step so the workflow's `try/catch` sees a task error. +#[sqlx::test(fixtures("base"))] +async fn a_child_failed_outside_the_worker_wakes_its_parent( + db: Pool, +) -> anyhow::Result<()> { + let child = Uuid::new_v4(); + let parent = plant_parked_parent(&db, &[("slowTask", child)]).await?; + let child_job = get_mini_completed_job(&child, W_ID, &db).await?.unwrap(); + + add_completed_job_error( + &db, + &child_job, + 0, + None, + json!({"name": "ExecutionErr", "message": "Job timed out after no ping"}), + "monitor", + false, + None, + ) + .await?; + + let (suspend, parked, status) = parent_state(&db, parent).await?; + assert_eq!(suspend, 0, "the parent must be released"); + assert!( + parked, + "suspend_until stays set: the suspended pull query keys on it" + ); + let step = &status["_checkpoint"]["completed_steps"]["slowTask"]; + assert_eq!(step["__wmill_error"], json!(true), "{status}"); + assert_eq!(step["child_job_id"], json!(child.to_string())); + assert_eq!( + step["result"]["error"]["message"], + json!("Job timed out after no ping") + ); + assert!( + status["_checkpoint"].get("pending_steps").is_none(), + "nothing left to wait on: {status}" + ); + assert!( + windmill_common::wac::WAC_SUSPEND_READY.swap(false, std::sync::atomic::Ordering::Relaxed) + ); + Ok(()) +} + +/// Only a child the parent is waiting on moves the counter. A child the body +/// launched itself, or a completion arriving after the key was re-dispatched to +/// another job, records its timeline entry and nothing else. +#[sqlx::test(fixtures("base"))] +async fn a_child_the_parent_is_not_waiting_on_leaves_it_parked( + db: Pool, +) -> anyhow::Result<()> { + let awaited = Uuid::new_v4(); + let parent = plant_parked_parent(&db, &[("task", awaited)]).await?; + let stray = Uuid::new_v4(); + insert_job(&db, stray, Some(parent)).await?; + + let stray_job = get_mini_completed_job(&stray, W_ID, &db).await?.unwrap(); + add_completed_job_error( + &db, + &stray_job, + 0, + None, + json!({"message": "boom"}), + "w", + false, + None, + ) + .await?; + + let (suspend, _, status) = parent_state(&db, parent).await?; + assert_eq!( + suspend, 1, + "a stray child must not release the parent: {status}" + ); + assert_eq!(status["_checkpoint"]["completed_steps"], json!({})); + assert!( + status[stray.to_string()]["duration_ms"].is_number(), + "the timeline entry is still stamped: {status}" + ); + + let awaited_job = get_mini_completed_job(&awaited, W_ID, &db).await?.unwrap(); + let result = serde_json::value::to_raw_value(&json!("done"))?; + add_completed_job( + &db, + &awaited_job, + true, + false, + Json(&result), + None, + 0, + None, + false, + None, + false, + ) + .await?; + + let (suspend, _, status) = parent_state(&db, parent).await?; + assert_eq!(suspend, 0); + assert_eq!( + status["_checkpoint"]["completed_steps"]["task"], + json!("done") + ); + Ok(()) +} diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs new file mode 100644 index 0000000000..5a83730d24 --- /dev/null +++ b/backend/tests/wac_suspend_started_at.rs @@ -0,0 +1,111 @@ +//! Guards what `suspend_wac_parent` promises: the `started_at` invariant documented on +//! it, the segment length it hands back for metering, and that it stands down for a +//! cancel already on the row. + +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_worker::wac_executor::{suspend_wac_parent, WacPark}; + +#[sqlx::test] +async fn wac_suspend_clears_started_at(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, started_at) \ + VALUES ($1, 'test-workspace', now(), true, now() - interval '4 days')", + ) + .bind(job_id) + .execute(&db) + .await?; + + let mut tx = db.begin().await?; + let WacPark::Parked(segment_ms) = + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await? + else { + panic!("an uncancelled parent must park"); + }; + tx.commit().await?; + + // The segment is what gets billed, so it must be the run that just ended, measured + // from the pull — not the park ahead of it, and not zero. + let four_days_ms = 4 * 24 * 3600 * 1000; + assert!( + segment_ms.is_some_and(|ms| (ms - four_days_ms).abs() < 60_000), + "expected the ended segment (~{four_days_ms}ms), got {segment_ms:?}" + ); + + let (started_at, running, suspend, suspend_until): ( + Option>, + bool, + i32, + Option>, + ) = sqlx::query_as( + "SELECT started_at, running, suspend, suspend_until FROM v2_job_queue WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + + assert_eq!( + started_at, None, + "a parked parent must not carry the previous segment's started_at" + ); + assert_eq!(suspend, 1); + assert!(suspend_until.is_some()); + assert!( + running, + "running stays true so the normal pull query skips the parked row" + ); + + Ok(()) +} + +/// A soft cancel sets `canceled_by` and `suspend = 0` and leaves acting on it to the next +/// pull. Parking over that holds the row until `suspend_until` — a whole day on a +/// `sleep(86400)` — so the park has to stand down and let the job complete instead. +#[sqlx::test] +async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job_queue \ + (id, workspace_id, scheduled_for, running, started_at, suspend, canceled_by, canceled_reason) \ + VALUES ($1, 'test-workspace', now(), true, now() - interval '30 seconds', 0, 'alice', 'no longer needed')", + ) + .bind(job_id) + .execute(&db) + .await?; + + let mut tx = db.begin().await?; + let parked = suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 86400.0).await?; + tx.commit().await?; + + match &parked { + WacPark::Cancelled(cancel) => { + assert_eq!(cancel.username.as_deref(), Some("alice")); + assert_eq!(cancel.reason.as_deref(), Some("no longer needed")); + } + other => panic!("a cancelled parent must not park, got {other:?}"), + } + + let (suspend, suspend_until, started_at): ( + i32, + Option>, + Option>, + ) = sqlx::query_as( + "SELECT suspend, suspend_until, started_at FROM v2_job_queue WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + + assert_eq!(suspend, 0, "the cancel's suspend = 0 must survive"); + assert_eq!( + suspend_until, None, + "a suspend_until would hold the row back for the whole park window" + ); + assert!( + started_at.is_some(), + "the segment ran, so its start must stay for the completion's duration" + ); + + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 2d3f1373fe..23d88301b7 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -318,19 +318,35 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho resp.text().await? ); } - // ...and the one `settings/global` key on the allowlist, which the CLI reads before - // creating a user on a git-sync push. `ws_base_url` is the control: the handler leaves - // it as ungated as `automate_username_creation`, so only the allowlist stops it. + // ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a + // git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler + // leaves it as ungated as these, so only the allowlist stops it. + for key in ["automate_username_creation", "uid", "hub_base_url"] { + let resp = authed( + client().get(format!("{api}/settings/global/{key}")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "WM_TOKEN must still read {key}: {}", + resp.text().await? + ); + } + // The same hub pull reads `hub_api_secret` for a private Hub, but a secret stays out of + // a job's reach even when the token borrows a superadmin. let resp = authed( - client().get(format!("{api}/settings/global/automate_username_creation")), - &user_wm, + client().get(format!("{api}/settings/global/hub_api_secret")), + &sa_wm, ) .send() .await?; - assert_eq!( - resp.status(), - 200, - "WM_TOKEN must still read automate_username_creation: {}", + let status = resp.status().as_u16(); + assert!( + status == 401 || status == 403, + "superadmin WM_TOKEN must not read hub_api_secret ({status}): {}", resp.text().await? ); let resp = authed( @@ -1094,6 +1110,7 @@ async fn test_privilege_gates_reject_a_job_token_directly( token_prefix: None, read_only: false, job_id, + credential_expiry: None, } } diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 419be6fdf3..b8bd6b30a7 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3091,16 +3091,16 @@ async fn test_php_job(db: Pool) -> anyhow::Result<()> { let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); - let content = r#" + let content = r#"// schema_validation ) -> anyhow: let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) - .email("test2@windmill.dev") + .as_user("test-user-2", "test2@windmill.dev") .run_until_complete(&db, false, server.addr.port()) .await; @@ -5635,6 +5649,83 @@ async fn test_whileloop_propagates_inner_iterator_eval_failure( Ok(()) } +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_whileloop_skip_if_evaluated_once_at_entry(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Regression test for #11007: `skip_if` on a while-loop module must be + // evaluated once, at loop entry, using the preceding step's result. + // Re-evaluating it on every iteration aliases `results.first` to the + // previous iteration's own result instead, which here lacks `.ok` and + // makes `skip_if` incorrectly turn true after the first iteration. + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "first", + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(): return {\"ok\": True}", + }, + }, + { + "id": "outer", + "value": { + "type": "whileloopflow", + "skip_failures": false, + "modules": [ + { + "id": "inner", + "value": { + "input_transforms": { + "i": { + "type": "javascript", + "expr": "flow_input.iter.index", + }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(i): return i", + }, + }, + ], + }, + "skip_if": { "expr": "!results.first.ok" }, + "stop_after_if": { + "expr": "result >= 2", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job).run_until_complete(&db, false, port).await; + + assert!(cjob.success, "flow should succeed"); + + let outer_module = get_module(&cjob, "outer").expect("outer module status"); + match outer_module { + windmill_common::flow_status::FlowStatusModule::Success { skipped, flow_jobs, .. } => { + assert!( + !skipped, + "while-loop must not be skipped: skip_if should only run once, at entry" + ); + assert_eq!( + flow_jobs.map(|v| v.len()), + Some(3), + "while-loop should run 3 iterations before stop_after_if halts it" + ); + } + other => panic!("expected outer module to be Success, got {other:?}"), + } + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall( diff --git a/backend/tests/ws_specific.rs b/backend/tests/ws_specific.rs index cc15161381..322de598b7 100644 --- a/backend/tests/ws_specific.rs +++ b/backend/tests/ws_specific.rs @@ -391,7 +391,8 @@ async fn test_create_resource_upsert_clears_ws_specific(db: Pool) -> a /// Regression for GHSA-xmr2-98m6-cjf7: a token scoped only to `resources:write:` /// must NOT use the resource-delete cascade to delete a linked secret variable it has -/// no `variables:write` scope for. +/// no `variables:write` scope for. The victim sits at a path the resource owns, which is +/// the only kind the cascade reaches at all. #[sqlx::test(fixtures("ws_specific"))] async fn test_scoped_token_cannot_cascade_delete_linked_variable( db: Pool, @@ -406,7 +407,7 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( "SECRET_TOKEN", ) .json(&json!({ - "path": "u/test-user/victim_secret", + "path": "u/test-user/db_victim_secret", "value": "hunter2", "is_secret": true, "description": "" @@ -421,7 +422,7 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( ) .json(&json!({ "path": "u/test-user/db", - "value": { "password": "$var:u/test-user/victim_secret" }, + "value": { "password": "$var:u/test-user/db_victim_secret" }, "resource_type": "object" })) .send() @@ -443,9 +444,206 @@ async fn test_scoped_token_cannot_cascade_delete_linked_variable( resp.text().await? ); assert!( - variable_exists(&db, "test-workspace", "u/test-user/victim_secret").await?, + variable_exists(&db, "test-workspace", "u/test-user/db_victim_secret").await?, "victim variable must survive the denied cascade" ); + // The scope check runs after the resource DELETE, so only the rollback keeps the resource + // alive — moving the check out of the transaction would silently delete it on a 403. + let resource_left: Option = + sqlx::query_scalar("SELECT COUNT(*) FROM resource WHERE workspace_id = $1 AND path = $2") + .bind("test-workspace") + .bind("u/test-user/db") + .fetch_one(&db) + .await?; + assert_eq!( + resource_left.unwrap_or(0), + 1, + "the denied delete must roll the resource back too" + ); + + Ok(()) +} + +/// Deleting a resource must not take a variable other things still need. Two gates, each +/// with a way past the other: a variable outside the resource's own path is never its to +/// delete, and even one it owns stays if another resource points at it. +#[sqlx::test(fixtures("ws_specific"))] +async fn test_resource_delete_spares_variables_it_does_not_own( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let create_var = |path: &'static str| { + authed( + client().post(format!("{base}/variables/create")), + "SECRET_TOKEN", + ) + .json(&json!({ "path": path, "value": "hunter2", "is_secret": true, "description": "" })) + .send() + }; + let create_res = |path: &'static str, var: &'static str| { + authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": path, + "value": { "password": format!("$var:{var}") }, + "resource_type": "object" + })) + .send() + }; + + // A shared secret at a path of its own, and two resources reading it. + assert_eq!(create_var("u/test-user/shared_canary").await?.status(), 201); + assert_eq!( + create_res("u/test-user/probe_a", "u/test-user/shared_canary") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/probe_b", "u/test-user/shared_canary") + .await? + .status(), + 201 + ); + + // A secret the resource at the same path owns, which a second resource also reads. + assert_eq!(create_var("u/test-user/owned").await?.status(), 201); + assert_eq!( + create_res("u/test-user/owned", "u/test-user/owned") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/borrower", "u/test-user/owned") + .await? + .status(), + 201 + ); + + for resource in ["u/test-user/probe_a", "u/test-user/owned"] { + let resp = authed( + client().delete(format!("{base}/resources/delete/{resource}")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "delete {resource}: {}", + resp.text().await? + ); + } + + assert!( + variable_exists(&db, "test-workspace", "u/test-user/shared_canary").await?, + "a variable the deleted resource only referenced must survive" + ); + assert!( + variable_exists(&db, "test-workspace", "u/test-user/owned").await?, + "an owned variable another resource still references must survive" + ); + + Ok(()) +} + +/// The bulk cascade follows what RLS actually deleted, not what the caller asked for: a +/// resource the request names but leaves standing neither cascades nor stops counting as a +/// referrer. Both halves matter, and neither covers the other. +#[sqlx::test(fixtures("ws_specific"))] +async fn test_bulk_delete_follows_what_rls_deleted(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + let create_var = |path: &'static str| { + authed( + client().post(format!("{base}/variables/create")), + "SECRET_TOKEN", + ) + .json(&json!({ "path": path, "value": "hunter2", "is_secret": true, "description": "" })) + .send() + }; + // ws_specific so the flag assertion at the end has something to check. + let create_res = |path: &'static str, var: &'static str| { + authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN", + ) + .json(&json!({ + "path": path, + "value": { "password": format!("$var:{var}") }, + "resource_type": "object", + "ws_specific": true + })) + .send() + }; + + // Private to test-user: a resource and the secret it owns. + assert_eq!(create_var("u/test-user/hidden_pwd").await?.status(), 201); + assert_eq!( + create_res("u/test-user/hidden", "u/test-user/hidden_pwd") + .await? + .status(), + 201 + ); + // test-user-2's own resource and secret, which the private resource above also reads. + assert_eq!(create_var("u/test-user-2/own_pwd").await?.status(), 201); + assert_eq!( + create_res("u/test-user-2/own", "u/test-user-2/own_pwd") + .await? + .status(), + 201 + ); + assert_eq!( + create_res("u/test-user/reader", "u/test-user-2/own_pwd") + .await? + .status(), + 201 + ); + + // test-user-2 may write the private secret but has no access to its resource at all. + sqlx::query( + "UPDATE variable SET extra_perms = '{\"u/test-user-2\": true}'::jsonb + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/hidden_pwd'", + ) + .execute(&db) + .await?; + + let resp = authed( + client().delete(format!("{base}/resources/delete_bulk")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "paths": ["u/test-user/hidden", "u/test-user-2/own", "u/test-user/reader"] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "bulk delete: {}", resp.text().await?); + + assert!( + variable_exists(&db, "test-workspace", "u/test-user/hidden_pwd").await?, + "the variable of a resource RLS refused to delete must survive" + ); + assert!( + variable_exists(&db, "test-workspace", "u/test-user-2/own_pwd").await?, + "a requested resource RLS left standing still counts as a referrer" + ); + // ws_specific has no RLS policy of its own, so clearing it by requested path rather than + // by deleted path would quietly turn a surviving resource workspace-generic. + assert_eq!( + ws_specific_row_count(&db, "test-workspace", "resource", "u/test-user/hidden").await?, + 1, + "a resource RLS refused to delete must keep its ws_specific flag" + ); Ok(()) } diff --git a/backend/windmill-ai/Cargo.toml b/backend/windmill-ai/Cargo.toml index 0276240246..98ea87a4e3 100644 --- a/backend/windmill-ai/Cargo.toml +++ b/backend/windmill-ai/Cargo.toml @@ -23,6 +23,7 @@ async-trait.workspace = true async-stream.workspace = true base64.workspace = true bytes.workspace = true +indexmap.workspace = true eventsource-stream.workspace = true futures.workspace = true http.workspace = true diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index bf16877f7b..a4dcbc4f35 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -24,6 +24,16 @@ lazy_static::lazy_static! { .ok() .map(|v| v == "true" || v == "1") .unwrap_or(false); + /// Drops the cache breakpoints from agent-step requests on every Anthropic platform, + /// not just the one that motivates it: a Google Cloud project can have explicit prompt + /// caching turned off (by request to Cloud support), and Vertex then rejects any request + /// carrying breakpoints. An instance that sets this to unblock such a project also gives + /// up caching on its direct-Anthropic and Foundry resources. + pub static ref DISABLE_ANTHROPIC_PROMPT_CACHING: bool = + std::env::var("DISABLE_ANTHROPIC_PROMPT_CACHING") + .ok() + .map(|v| v == "true" || v == "1") + .unwrap_or(false); } pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index 69c8ac4538..599cf0cca3 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -210,7 +210,9 @@ pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), return Err(format!("Price override for {} is not an object", key)); }; for field in ["input", "output", "cache_read", "cache_write"] { - let Some(rate) = price.get(field) else { continue }; + let Some(rate) = price.get(field) else { + continue; + }; let rate = rate .as_f64() .filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE); @@ -223,9 +225,98 @@ pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), } for required in ["input", "output"] { if !price.contains_key(required) { - return Err(format!("Price override for {} is missing {}", key, required)); + return Err(format!( + "Price override for {} is missing {}", + key, required + )); } } } Ok(()) } + +// ============================================================================ +// Per-model token maps +// ============================================================================ + +/// One `provider:model` -> tokens map of the AI config, with the bounds the API +/// documents for it. Both maps are validated the same way, so the bounds live here +/// rather than at each call site. +pub struct TokenMap { + pub field: &'static str, + pub label: &'static str, + pub min: i64, + pub max: i64, +} + +/// The chat compacts its history as it nears the window, so a value below any real +/// model's window would compact every turn; the ceiling is far above any shipped +/// model and only catches typos. +pub const CONTEXT_WINDOWS: TokenMap = TokenMap { + field: "context_window_per_model", + label: "Context window", + min: 1024, + max: 10_000_000, +}; + +/// The completion cap the chat asks the provider for. +pub const OUTPUT_LIMITS: TokenMap = + TokenMap { field: "max_tokens_per_model", label: "Output limit", min: 1, max: 2_000_000 }; + +pub const TOKEN_MAPS: [TokenMap; 2] = [CONTEXT_WINDOWS, OUTPUT_LIMITS]; + +pub fn validate_token_limit(map: &TokenMap, key: &str, tokens: i64) -> Result<(), String> { + if (map.min..=map.max).contains(&tokens) { + Ok(()) + } else { + Err(format!( + "{} for {} must be between {} and {} tokens", + map.label, key, map.min, map.max + )) + } +} + +/// Bound these maps in the untyped instance AI config, for the reason given on +/// `validate_model_pricing_json`. +pub fn validate_token_maps_json(ai_config: &serde_json::Value) -> Result<(), String> { + for map in TOKEN_MAPS.iter() { + let entries = match ai_config.get(map.field) { + None | Some(serde_json::Value::Null) => continue, + Some(v) => v + .as_object() + .ok_or_else(|| format!("{} must be an object", map.field))?, + }; + for (key, tokens) in entries { + let tokens = tokens + .as_i64() + .ok_or_else(|| format!("{} for {} must be an integer", map.label, key))?; + validate_token_limit(map, key, tokens)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn instance_token_maps_are_bounded_integers() { + assert!(validate_token_maps_json(&json!({})).is_ok()); + for map in TOKEN_MAPS.iter() { + let entry = |value| json!({ map.field: { "customai:m": value } }); + assert!(validate_token_maps_json(&entry(json!(map.min))).is_ok()); + assert!(validate_token_maps_json(&entry(json!(map.max))).is_ok()); + for bad in [ + json!(map.min - 1), + json!(map.max + 1), + json!(4096.5), + json!("4096"), + ] { + assert!(validate_token_maps_json(&entry(bad)).is_err()); + } + assert!(validate_token_maps_json(&json!({ map.field: [4096] })).is_err()); + } + } +} diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index f22f28de14..0bb7316dcb 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -1,7 +1,7 @@ use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_google::parse_data_url, - ai_providers::{AIPlatform, AIProvider}, + ai_providers::{AIPlatform, AIProvider, DISABLE_ANTHROPIC_PROMPT_CACHING}, image_handler::prepare_messages_for_api, proxy::{ add_user_to_body, common_outbound_headers, credential_header, ProxyBuildArgs, ProxyRequest, @@ -632,15 +632,13 @@ impl AnthropicQueryBuilder { } } + let caching = !*DISABLE_ANTHROPIC_PROMPT_CACHING; + let system = collect_system_prompt(&prepared_messages, args.system_prompt).map(|text| { vec![AnthropicSystemContent { r#type: "text".to_string(), text, - cache_control: if self.is_vertex() { - None - } else { - Some(CacheControl::ephemeral()) - }, + cache_control: caching.then(CacheControl::ephemeral), }] }); @@ -665,7 +663,7 @@ impl AnthropicQueryBuilder { let max_tokens = Some(args.max_tokens.unwrap_or(64000)); // Apply cache_control on the last custom tool - if !self.is_vertex() { + if caching { if let Some(ref mut tools_vec) = tools_option { if let Some(AnthropicTool::Custom(ref mut custom)) = tools_vec.last_mut() { custom.cache_control = Some(CacheControl::ephemeral()); @@ -674,7 +672,7 @@ impl AnthropicQueryBuilder { } // Apply cache_control on the last content block of the last message - if !self.is_vertex() { + if caching { if let Some(last_msg) = anthropic_messages.last_mut() { if let Some(last_block) = last_msg.content.last_mut() { match last_block { @@ -769,6 +767,7 @@ impl QueryBuilder for AnthropicQueryBuilder { let AnthropicSSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, events_str, annotations, @@ -792,6 +791,7 @@ impl QueryBuilder for AnthropicQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, @@ -882,10 +882,15 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + async fn build_text_body_on( + platform: AIPlatform, + messages: &[OpenAIMessage], + system_prompt: Option<&str>, + tools: Option<&[ToolDef]>, + ) -> String { let args = BuildRequestArgs { messages, - tools: None, + tools, model: "claude-sonnet-4", temperature: None, reasoning_effort: None, @@ -897,14 +902,19 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: None, + reasoning_summary: false, }; - AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard) + AnthropicQueryBuilder::new(AIProvider::Anthropic, platform) .build_request(&args, &authed_client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_text_body_on(AIPlatform::Standard, messages, system_prompt, None).await + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] @@ -946,6 +956,35 @@ mod tests { assert!(request.get("system").is_none()); } + /// Vertex serves the same Messages API and honours `cache_control` breakpoints, so + /// its requests must carry the same three the standard platform gets. + #[tokio::test] + async fn sets_cache_breakpoints_on_every_platform() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + let tools = vec![ToolDef { + r#type: "function".to_string(), + function: ToolDefFunction { + name: "get_weather".to_string(), + description: None, + parameters: RawValue::from_string("{}".to_string()).unwrap(), + }, + }]; + let ephemeral = serde_json::json!({ "type": "ephemeral" }); + + for platform in [AIPlatform::Standard, AIPlatform::GoogleVertexAi] { + let body = + build_text_body_on(platform, &messages, Some(SYSTEM_PROMPT), Some(&tools)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["cache_control"], ephemeral); + let sent_tools = request["tools"].as_array().unwrap(); + assert_eq!(sent_tools.last().unwrap()["cache_control"], ephemeral); + let sent = request["messages"].as_array().unwrap(); + let content = sent.last().unwrap()["content"].as_array().unwrap(); + assert_eq!(content.last().unwrap()["cache_control"], ephemeral); + } + } + fn has_header(headers: &[(String, String)], name: &str, value: &str) -> bool { headers .iter() diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index cf977f2484..412e057396 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -1074,7 +1074,8 @@ impl BedrockQueryBuilder { let mut accumulated_text = String::new(); let mut events_str = String::new(); - let mut accumulated_tool_calls: HashMap = HashMap::new(); + let mut accumulated_tool_calls: indexmap::IndexMap = + indexmap::IndexMap::new(); let mut current_tool_use_id: Option = None; let mut usage: Option = None; // Claude reasoning block for the turn (only populated when thinking is on), @@ -1188,6 +1189,13 @@ impl BedrockQueryBuilder { Some(accumulated_text) }; + // The block folded for replay is also what the reader sees as thinking. Read out + // before the block itself moves into the tool calls below. + let reasoning_text = reasoning + .as_ref() + .and_then(|r| r.reasoning_text.clone()) + .filter(|t| !t.is_empty()); + let tool_calls = streaming_tool_calls_to_openai( accumulated_tool_calls.into_values().collect(), reasoning, @@ -1195,6 +1203,7 @@ impl BedrockQueryBuilder { Ok(ParsedResponse::Text { content, + reasoning: reasoning_text, tool_calls, events_str: if events_str.is_empty() { None @@ -1255,7 +1264,10 @@ mod tests { // recovers the uncached share by subtracting the details back out. assert_eq!(usage["usage"]["prompt_tokens"], 1010); assert_eq!(usage["usage"]["completion_tokens"], 7); - assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!( + usage["usage"]["prompt_tokens_details"]["cached_tokens"], + 900 + ); assert_eq!( usage["usage"]["prompt_tokens_details"]["cache_write_tokens"], 100 diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 21428903fa..8320ef5661 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -666,6 +666,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { let GeminiSSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, mut events_str, stream_event_processor, @@ -698,6 +699,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index 34185a8852..14c9fdb4d3 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -1,6 +1,7 @@ use crate::{ ai_providers::AIProvider, ai_types::OpenAIToolCall, + credentials::ProviderCredentials, image_handler::{prepare_messages_for_api, s3_object_to_content_part}, proxy::{build_openai_compatible_proxy_request, ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, @@ -11,7 +12,14 @@ use crate::{ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{client::AuthedClient, error::Error}; +use std::{ + collections::{BTreeMap, HashMap}, + hash::{DefaultHasher, Hash, Hasher}, + time::{Duration, Instant}, +}; +use windmill_common::{cache::Cache, client::AuthedClient, error::Error}; + +use super::REASONING_OFF_SENTINEL; // Responses API structures #[derive(Deserialize)] @@ -192,13 +200,79 @@ pub struct ResponsesApiTextFormat { pub format: ResponsesApiTextFormatConfig, } -/// Reasoning config for the Responses API (`reasoning: { effort }`). -/// The summary is intentionally not requested, mirroring the copilot chat: OpenAI -/// gates reasoning summaries behind organization verification, so asking for one -/// would fail the request for unverified orgs. +/// Reasoning config for the Responses API (`reasoning: { effort, summary }`). #[derive(Serialize)] pub struct ResponsesApiReasoning { pub effort: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +lazy_static::lazy_static! { + /// Refused reasoning summaries, so later requests skip asking instead of paying a + /// rejected call each. A refusal belongs to the organization the request bills or to the + /// model, so the key holds the model and everything that authenticates (the API key and + /// the resource headers, which can carry it instead). Entries expire as an org gets verified. + static ref REASONING_SUMMARY_UNAVAILABLE: Cache<(String, u64), Instant> = Cache::new(500); +} + +const REASONING_SUMMARY_UNAVAILABLE_TTL: Duration = Duration::from_secs(3600); + +fn reasoning_summary_cache_key( + base_url: &str, + model: &str, + api_key: Option<&str>, + custom_headers: &HashMap, +) -> (String, u64) { + let mut hasher = DefaultHasher::new(); + model.hash(&mut hasher); + api_key.hash(&mut hasher); + // Sorted: two maps with the same entries can iterate them in different orders. + custom_headers + .iter() + .collect::>() + .hash(&mut hasher); + (base_url.to_string(), hasher.finish()) +} + +fn credentials_cache_key(credentials: &ProviderCredentials, model: &str) -> (String, u64) { + reasoning_summary_cache_key( + &credentials.base_url, + model, + credentials.api_key.as_deref(), + &credentials.custom_headers, + ) +} + +/// Whether this model is known to be refused reasoning summaries with these credentials. +pub fn is_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) -> bool { + REASONING_SUMMARY_UNAVAILABLE + .get(&credentials_cache_key(credentials, model)) + .is_some_and(|learned_at| learned_at.elapsed() < REASONING_SUMMARY_UNAVAILABLE_TTL) +} + +/// Record that this model was refused a reasoning summary with these credentials. +pub fn remember_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) { + REASONING_SUMMARY_UNAVAILABLE.insert(credentials_cache_key(credentials, model), Instant::now()); +} + +/// Whether a rejected request was refused over its reasoning summary, e.g. `Your +/// organization must be verified to generate reasoning summaries` (param +/// `reasoning.summary`). An OpenAI-kind resource can also point at a gateway that validates +/// the body strictly and names only the unknown `summary` property. +pub fn rejects_reasoning_summary(status: u16, body: &str) -> bool { + // 422 is how FastAPI-based gateways reject a body that fails validation. + if !matches!(status, 400 | 403 | 422) { + return false; + } + let body = body.to_lowercase(); + let unknown_field = body.contains("additional properties are not allowed") + || body.contains("unrecognized request argument") + || body.contains("extra inputs are not permitted"); + body.contains("reasoning.summary") + || body.contains("verified to generate reasoning summar") + || body.contains("verified to stream reasoning summar") + || (unknown_field && body.contains("summary")) } #[derive(Serialize)] @@ -435,9 +509,13 @@ impl OpenAIQueryBuilder { tools, stream: Some(true), temperature: args.temperature, - reasoning: args - .reasoning_effort - .map(|effort| ResponsesApiReasoning { effort: effort.to_string() }), + reasoning: args.reasoning_effort.map(|effort| ResponsesApiReasoning { + effort: effort.to_string(), + // A request that does not reason has nothing to summarize, yet asking still + // gets an unverified organization's request rejected. + summary: (args.reasoning_summary && effort != REASONING_OFF_SENTINEL) + .then(|| "auto".to_string()), + }), max_output_tokens: args.max_tokens, text, prompt_cache_key: args.prompt_cache_key, @@ -538,6 +616,8 @@ impl QueryBuilder for OpenAIQueryBuilder { } else { Some(parser.accumulated_content) }, + reasoning: (!parser.accumulated_reasoning.is_empty()) + .then_some(parser.accumulated_reasoning), tool_calls: parser.accumulated_tool_calls.into_values().collect(), events_str: Some(parser.events_str), annotations: parser.annotations, @@ -637,8 +717,11 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { - let args = BuildRequestArgs { + fn text_args<'a>( + messages: &'a [OpenAIMessage], + system_prompt: Option<&'a str>, + ) -> BuildRequestArgs<'a> { + BuildRequestArgs { messages, tools: None, model: "gpt-5", @@ -652,14 +735,98 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: Some(PROMPT_CACHE_KEY), - }; + reasoning_summary: true, + } + } + async fn build_body(args: &BuildRequestArgs<'_>) -> String { OpenAIQueryBuilder::new(AIProvider::OpenAI) - .build_request(&args, &client(), "test-workspace") + .build_request(args, &client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_body(&text_args(messages, system_prompt)).await + } + + async fn reasoning_of(effort: Option<&str>, reasoning_summary: bool) -> serde_json::Value { + let messages = vec![message("user", "hi")]; + let args = BuildRequestArgs { + reasoning_effort: effort, + reasoning_summary, + ..text_args(&messages, None) + }; + let request: serde_json::Value = serde_json::from_str(&build_body(&args).await).unwrap(); + request["reasoning"].clone() + } + + #[tokio::test] + async fn requests_a_reasoning_summary_only_when_the_model_reasons() { + assert_eq!( + reasoning_of(Some("high"), true).await, + serde_json::json!({ "effort": "high", "summary": "auto" }) + ); + assert_eq!( + reasoning_of(Some("none"), true).await, + serde_json::json!({ "effort": "none" }) + ); + assert_eq!( + reasoning_of(Some("high"), false).await, + serde_json::json!({ "effort": "high" }) + ); + assert!(reasoning_of(None, true).await.is_null()); + } + + #[test] + fn recognizes_a_refused_reasoning_summary() { + let unverified = r#"{"error":{"message":"Your organization must be verified to generate reasoning summaries. Please go to: https://platform.openai.com/settings/organization/general and click on Verify Organization.","type":"invalid_request_error","param":"reasoning.summary","code":"unsupported_value"}}"#; + assert!(rejects_reasoning_summary(400, unverified)); + assert!(!rejects_reasoning_summary(500, unverified)); + assert!(rejects_reasoning_summary( + 400, + r#"{"detail":"Additional properties are not allowed ('summary' was unexpected)"}"# + )); + assert!(rejects_reasoning_summary( + 422, + r#"{"detail":[{"type":"extra_forbidden","loc":["body","reasoning","summary"],"msg":"Extra inputs are not permitted","input":"auto"}]}"# + )); + assert!(!rejects_reasoning_summary( + 400, + r#"{"error":{"message":"Invalid 'prompt_cache_key': string too long","param":"prompt_cache_key"}}"# + )); + } + + /// A resource can authenticate through `headers` with no API key: one organization's + /// refusal must not withhold summaries from another's. + #[test] + fn keys_a_refused_summary_by_the_header_credential() { + let headers = |pairs: &[(&str, &str)]| { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>() + }; + let url = "https://api.openai.com/v1"; + let org_a = headers(&[("Authorization", "Bearer org-a"), ("X-Trace", "1")]); + let org_a_reordered = headers(&[("X-Trace", "1"), ("Authorization", "Bearer org-a")]); + let org_b = headers(&[("Authorization", "Bearer org-b"), ("X-Trace", "1")]); + + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_b) + ); + assert_eq!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_a_reordered) + ); + // A model can refuse summaries that another model on the same credentials streams. + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5-mini", None, &org_a) + ); + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index bc2d8f2992..b6040d8d79 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -251,6 +251,7 @@ impl QueryBuilder for OtherQueryBuilder { let OpenAISSEParser { accumulated_content, + accumulated_reasoning, accumulated_tool_calls, mut events_str, stream_event_processor, @@ -277,6 +278,7 @@ impl QueryBuilder for OtherQueryBuilder { } else { Some(accumulated_content) }, + reasoning: (!accumulated_reasoning.is_empty()).then_some(accumulated_reasoning), tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations: Vec::new(), diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index c8f5fbb092..3281b51cd0 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -27,12 +27,17 @@ pub struct BuildRequestArgs<'a> { /// the prefix (the step), never from the request. `None` retries a key the /// endpoint rejected. pub prompt_cache_key: Option<&'a str>, + /// Ask for a summary of the model's reasoning where the provider streams one. + /// `false` once the provider refused summaries to these credentials. + pub reasoning_summary: bool, } /// Response from AI provider pub enum ParsedResponse { Text { content: Option, + /// The thinking the model streamed before the answer, when it emitted any. + reasoning: Option, tool_calls: Vec, events_str: Option, annotations: Vec, diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 96692d0173..57acd66738 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use eventsource_stream::Eventsource; +use indexmap::IndexMap; use reqwest::Response; use serde::Deserialize; use tokio_stream::StreamExt; @@ -135,7 +136,11 @@ pub trait SSEParser { pub struct OpenAISSEParser { pub accumulated_content: String, - pub accumulated_tool_calls: HashMap, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, + // Insertion-ordered in every parser: tool calls run and are persisted in the order the + // stream showed them, and a chat attaches a round's thinking to its first call. + pub accumulated_tool_calls: IndexMap, pub events_str: String, pub stream_event_processor: Box, /// Token usage from final chunk (when stream_options.include_usage is true) @@ -146,7 +151,8 @@ impl OpenAISSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), - accumulated_tool_calls: HashMap::new(), + accumulated_reasoning: String::new(), + accumulated_tool_calls: IndexMap::new(), events_str: String::new(), stream_event_processor, usage: None, @@ -175,6 +181,7 @@ impl SSEParser for OpenAISSEParser { if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { if let Some(delta) = choices.remove(0).delta { if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { + self.accumulated_reasoning.push_str(&reasoning); let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; self.stream_event_processor .send(event, &mut self.events_str) @@ -353,7 +360,9 @@ enum ContentBlockState { /// Anthropic SSE Parser for streaming responses pub struct AnthropicSSEParser { pub accumulated_content: String, - pub accumulated_tool_calls: HashMap, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, + pub accumulated_tool_calls: IndexMap, pub events_str: String, pub stream_event_processor: Box, /// Track content block types by index @@ -375,7 +384,8 @@ impl AnthropicSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), - accumulated_tool_calls: HashMap::new(), + accumulated_reasoning: String::new(), + accumulated_tool_calls: IndexMap::new(), events_str: String::new(), stream_event_processor, content_blocks: HashMap::new(), @@ -455,6 +465,7 @@ impl SSEParser for AnthropicSSEParser { .thinking .get_or_insert_with(String::new) .push_str(&thinking); + self.accumulated_reasoning.push_str(&thinking); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: thinking }, @@ -523,6 +534,7 @@ impl SSEParser for AnthropicSSEParser { .thinking .get_or_insert_with(String::new) .push_str(&thinking); + self.accumulated_reasoning.push_str(&thinking); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: thinking }, @@ -590,7 +602,9 @@ impl SSEParser for AnthropicSSEParser { /// `windmill_ai::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, - pub accumulated_tool_calls: HashMap, + /// The thinking streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, + pub accumulated_tool_calls: IndexMap, pub events_str: String, pub stream_event_processor: Box, tool_call_index: i64, @@ -603,7 +617,8 @@ impl GeminiSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), - accumulated_tool_calls: HashMap::new(), + accumulated_reasoning: String::new(), + accumulated_tool_calls: IndexMap::new(), events_str: String::new(), stream_event_processor, tool_call_index: 0, @@ -621,6 +636,7 @@ impl SSEParser for GeminiSSEParser { }; if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { + self.accumulated_reasoning.push_str(&reasoning); self.stream_event_processor .send( StreamingEvent::ReasoningTokenDelta { content: reasoning }, @@ -802,6 +818,14 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.output_text.annotation.added")] AnnotationAdded { annotation: OpenAIUrlCitationEvent }, + /// A new reasoning summary part starts (only sent when `reasoning.summary` was requested) + #[serde(rename = "response.reasoning_summary_part.added")] + ReasoningSummaryPartAdded {}, + + /// Reasoning summary text delta + #[serde(rename = "response.reasoning_summary_text.delta")] + ReasoningSummaryTextDelta { delta: String }, + /// Catch-all for unknown event types #[serde(other)] Other, @@ -810,7 +834,9 @@ pub enum OpenAIResponsesSSEEvent { /// OpenAI Responses API SSE Parser for streaming responses pub struct OpenAIResponsesSSEParser { pub accumulated_content: String, - pub accumulated_tool_calls: HashMap, + /// The reasoning summary streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, + pub accumulated_tool_calls: IndexMap, /// Maps item_id -> (name, call_id) for function calls tool_call_metadata: HashMap, /// Maps item_id -> accumulated arguments @@ -823,13 +849,16 @@ pub struct OpenAIResponsesSSEParser { pub used_websearch: bool, /// Token usage from response.completed event pub usage: Option, + /// Reasoning summary parts seen so far, to separate them as paragraphs + reasoning_summary_parts: usize, } impl OpenAIResponsesSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), - accumulated_tool_calls: HashMap::new(), + accumulated_reasoning: String::new(), + accumulated_tool_calls: IndexMap::new(), tool_call_metadata: HashMap::new(), tool_call_arguments: HashMap::new(), events_str: String::new(), @@ -837,6 +866,7 @@ impl OpenAIResponsesSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + reasoning_summary_parts: 0, } } } @@ -946,6 +976,28 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { + self.reasoning_summary_parts += 1; + if self.reasoning_summary_parts > 1 { + self.accumulated_reasoning.push_str("\n\n"); + let event = + StreamingEvent::ReasoningTokenDelta { content: "\n\n".to_string() }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + + OpenAIResponsesSSEEvent::ReasoningSummaryTextDelta { delta } => { + if !delta.is_empty() { + self.accumulated_reasoning.push_str(&delta); + let event = StreamingEvent::ReasoningTokenDelta { content: delta }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + // Ignore other event types OpenAIResponsesSSEEvent::Done {} | OpenAIResponsesSSEEvent::Created {} @@ -1002,6 +1054,33 @@ mod tests { assert_eq!(token_usage.total_tokens, Some(4821)); } + struct ReasoningSink; + + #[async_trait::async_trait] + impl StreamEventSink for ReasoningSink { + async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> { + if let StreamingEvent::ReasoningTokenDelta { content } = event { + events_str.push_str(&content); + } + Ok(()) + } + } + + #[tokio::test] + async fn streams_openai_responses_reasoning_summary_parts_as_paragraphs() { + let mut parser = OpenAIResponsesSSEParser::new(Box::new(ReasoningSink)); + for data in [ + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":0,"delta":"**Planning**"}"#, + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":1,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":1,"delta":"Then answer"}"#, + ] { + parser.parse_event_data(data).await.unwrap(); + } + assert_eq!(parser.events_str, "**Planning**\n\nThen answer"); + assert_eq!(parser.accumulated_reasoning, "**Planning**\n\nThen answer"); + } + #[test] fn openai_delta_parses_reasoning_content() { // DeepSeek and similar stream reasoning under `reasoning_content`. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 6d231dc675..12e06a35d8 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -78,17 +78,50 @@ impl Default for OutputType { #[serde(tag = "kind", rename_all = "lowercase")] pub enum Memory { Off, - Auto { - #[serde(default)] + Window { + #[serde(default, deserialize_with = "deserialize_null_as_zero")] context_length: usize, - #[serde(default)] + }, + /// Written before `window`. Its `memory_id` stays a fallback behind the run's memory id. + Auto { + #[serde(default, deserialize_with = "deserialize_null_as_zero")] + context_length: usize, + #[serde(default, deserialize_with = "deserialize_blank_as_none")] memory_id: Option, }, + /// Written before a step had history inputs of its own, and read on its own where it remains. Manual { messages: Vec, }, } +// An editor form can leave `""` in a legacy baked id it never filled; it means no id rather than +// failing every run of the step. +fn deserialize_blank_as_none<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + match as serde::Deserialize>::deserialize(deserializer)? { + Some(id) if !id.trim().is_empty() => Uuid::parse_str(id.trim()) + .map(Some) + .map_err(serde::de::Error::custom), + _ => Ok(None), + } +} + +// A count the editor's number field was cleared of is stored as `null`, which `default` does not +// cover; it reads as 0, memory off, rather than failing every run of the step. +fn deserialize_null_as_zero<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result { + as serde::Deserialize>::deserialize(deserializer).map(Option::unwrap_or_default) +} + +fn deserialize_present<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + ::deserialize(deserializer).map(Some) +} + #[derive(Deserialize)] struct AIAgentArgsRaw { provider: ProviderWithResource, @@ -103,6 +136,13 @@ struct AIAgentArgsRaw { streaming: Option, max_iterations: Option, memory: Option, + // A null must stay distinguishable from an absent key: a step whose own memory id evaluates to + // nothing runs stateless instead of falling back to the run's memory id. + #[serde(default, deserialize_with = "deserialize_present")] + memory_id: Option, + #[serde(default)] + previous_messages: Option>, + enabled_tools: Option>, // Legacy field for backward compatibility messages_context_length: Option, #[serde(default)] @@ -123,6 +163,13 @@ pub struct AIAgentArgs { pub streaming: Option, pub max_iterations: Option, pub memory: Option, + /// Memory id set on the step, overriding the run's. Empty when its expression produced none. + pub memory_id: Option, + /// History supplied by the flow, replayed without reading or writing memory. + pub previous_messages: Option>, + /// Which of the agent's tools this run may call; `narrow_roster` holds what the names are and + /// what `None` means. + pub enabled_tools: Option>, pub credentials_check: bool, } @@ -135,12 +182,17 @@ impl From for AIAgentArgs { }); // Backward compatibility: if context_length is 0, use off mode - let memory = memory.map(|memory| { - if let Memory::Auto { context_length: 0, .. } = memory { + let memory = memory.map(|memory| match memory { + Memory::Auto { context_length: 0, .. } | Memory::Window { context_length: 0 } => { Memory::Off - } else { - memory } + memory => memory, + }); + + let memory_id = raw.memory_id.map(|value| match value { + serde_json::Value::Null => String::new(), + serde_json::Value::String(s) => s.trim().to_string(), + value => value.to_string(), }); AIAgentArgs { @@ -155,6 +207,9 @@ impl From for AIAgentArgs { streaming: raw.streaming, max_iterations: raw.max_iterations, memory, + memory_id, + previous_messages: raw.previous_messages, + enabled_tools: raw.enabled_tools, credentials_check: raw.credentials_check.unwrap_or(false), } } @@ -369,6 +424,12 @@ pub struct AIAgentResult<'a> { pub messages: Vec>, #[serde(skip_serializing_if = "Option::is_none")] pub wm_stream: Option, + /// The model's thinking across every iteration of the loop, in order, blank-line + /// separated. Present whenever the provider's parser surfaced any, whether or not + /// the step streams, so a downstream step never has to pick it out of `wm_stream`. + /// Absent when the model thought nothing. + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] pub usage: Option, } diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 09720b2203..89e3c7c6b2 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -6,6 +6,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use windmill_common::{ assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, @@ -13,7 +14,9 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_filter, build_scope_path_predicate, ApiAuthed, ScopePathFilter, +}; // Partition-range backfill preview. The logic (producer resolution, range // enumeration, status join) is enterprise: the `private` build compiles the @@ -33,6 +36,7 @@ pub fn workspaced_service() -> Router { .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) .route("/graph", get(asset_graph)) + .route("/column_lineage", get(dbt_column_lineage)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) .route("/partitions_in_range", get(list_partitions_in_range)) @@ -663,10 +667,21 @@ struct DbtAssetProvenance { description: Option, #[serde(skip_serializing_if = "Vec::is_empty")] data_tests: Vec, - /// Declared column metadata (name -> description). NOT column lineage — - /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + /// Declared column metadata (name -> description): what `manifest.json` + /// carries, which is only the columns an author wrote down. #[serde(skip_serializing_if = "Option::is_none")] columns: Option, + /// Every column of the relation, typed and in order — + /// `[{"name": …, "type": …}]` — from the engine's static analysis. Present + /// only for a project that opted into it. + /// + /// Gated exactly like `columns` and the model's SQL: a full column list is + /// the shape of what the author WROTE, one level finer than the `ref()` + /// graph, which is ungated only because it draws relations the caller + /// already sees in `asset`. Widening that boundary has to be a decision, not + /// a consequence of a project turning the analysis pass on. + #[serde(skip_serializing_if = "Option::is_none")] + column_schema: Option, /// A source's declared freshness policy, for the staleness chip. #[serde(skip_serializing_if = "Option::is_none")] freshness: Option, @@ -946,6 +961,458 @@ struct DbtLineageEdge { to_asset_path: String, } +/// One column-to-column edge, in the same terms: the two relations and the two +/// columns, never dbt's node ids. +#[derive(Serialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct DbtColumnLineageEdge { + from_asset_path: String, + from_column: String, + to_asset_path: String, + to_column: String, + /// dbt's own word for how the value travelled: `copy` (passthrough), `mod` + /// (transformed), `scan` (read to produce the ROW rather than the value — a + /// join key, a predicate, a `group by`). Sent verbatim, including a kind + /// this engine version invented, because the renderer decides what a kind + /// means and the set is the engine's. + kind: String, +} + +/// The dbt relations a view is tracing, and which stored graph to read them +/// from. +/// +/// Several relations, answered as one union, because ONE selection reaches +/// several: a script's output column can be derived from columns of several dbt +/// models, and a model's columns can be consumed by scripts that feed others. +/// Asking per relation instead is a request per boundary plus the bookkeeping to +/// stitch the answers together and decide which of them is still current — which +/// is a cache, and is what taking them together exists to not need. +pub struct ColumnLineageQuery { + /// The `dbt://` relations whose lineage to return. + pub asset_paths: Vec, + /// The deployed version a view is drawing, when it is drawing one — the dbt + /// editor, which shows a single project as of a single deploy. + /// + /// A version-pinned answer is that version's project ALONE, the same as a + /// job-pinned one: the pin exists so the trace describes the stored graph on + /// screen, and another project's live graph is not part of it. Only the + /// unpinned answer crosses projects. + /// + /// A run's or an editor buffer's graph is NOT reachable from here: it pins + /// to a job, and that costs the job-read gate. + pub dbt_script_hash: Option, +} + +impl ColumnLineageQuery { + /// Built from the raw pairs rather than deserialized as a struct, because + /// `asset_path` REPEATS and `serde_urlencoded` — what `Query` deserializes + /// with — reads no sequence from a repeated key. A GET rather than a POST + /// body carrying the list: the method decides a scoped token's action, so a + /// POST would ask `assets:write` for a read and refuse a read-only token + /// outright. + pub fn from_query_pairs(pairs: Vec<(String, String)>) -> windmill_common::error::Result { + let mut asset_paths: Vec = Vec::new(); + let mut dbt_script_hash = None; + for (key, value) in pairs { + match key.as_str() { + "asset_path" => asset_paths.push(value), + // Hex, like every other script-hash parameter, so a page can + // pass `job.script_hash` verbatim. + "dbt_script_hash" => { + dbt_script_hash = + Some(serde_json::from_value(Value::String(value)).map_err(|_| { + windmill_common::error::Error::BadRequest( + "dbt_script_hash is not a script hash".to_string(), + ) + })?) + } + _ => {} + } + } + // REFUSED, not answered empty. A caller that named no relation — or + // misspelled the parameter — asked for something, and an empty component + // is what a relation with no lineage returns, so answering that way says + // "this has none" for a question that was never asked. + if asset_paths.is_empty() { + return Err(windmill_common::error::Error::BadRequest( + "at least one asset_path is required".to_string(), + )); + } + if asset_paths.len() > MAX_ASKED_RELATIONS { + return Err(windmill_common::error::Error::BadRequest(format!( + "at most {MAX_ASKED_RELATIONS} asset_path values may be asked about at once" + ))); + } + Ok(ColumnLineageQuery { asset_paths, dbt_script_hash }) + } +} + +#[derive(Serialize, Debug, PartialEq, Eq)] +pub struct ColumnLineageResponse { + /// Direct (`copy` / `mod`) column edges of the component the asked-for + /// relations' columns sit in, in the terms the canvas draws. Empty when no + /// project involved asked for the analysis pass, which is the ordinary case. + edges: Vec, + /// The component reaches further than what is here: `edges` holds the part + /// nearest the asked-for relations. Said rather than silently cut, because a + /// trace that stops short is otherwise indistinguishable from one that ends. + truncated: bool, +} + +/// How many edges one trace may carry back. The renderer draws a box per column, +/// so a component past this is unreadable however it is served — a synthetic +/// 3000-model project whose models share a column returns 58k direct edges and +/// 7.3MB. Applied over the walk, which is the LAST filter, so what survives is +/// the part nearest the selection rather than an arbitrary slice of it. +const MAX_TRACE_EDGES: usize = 5_000; + +/// How many times one trace may discover a project it has not read yet. Each +/// round costs a gate and a fetch, and a component crossing this many projects +/// has already outgrown what the canvas can show; stopping says what the edge +/// bound says. +const MAX_OWNER_ROUNDS: usize = 8; + +/// How many edges a trace may already hold before it stops looking for projects +/// it has not read. NOT a bound on what one trace fetches: the seeds' own +/// projects are read whole whatever their size, because reading them IS the +/// answer, and a project's edges arrive whole in any case — the walk is what +/// decides which of them are in the component, so a `LIMIT` would cut a set that +/// need not contain the asked-for relation at all. What a single fetch is +/// bounded by is the ingest's `MAX_COLUMN_EDGES` per version. This bounds the +/// EXPANSION on top of that: reading a further project's worth once a trace is +/// already this size buys nothing the walk will not cut at `MAX_TRACE_EDGES`. +const EXPANSION_EDGE_BUDGET: usize = 100_000; + +/// How many relations one request may ask about. Generous: the pipeline page +/// sends every dbt relation the selection's own producer lineage reaches, which +/// is a handful even in a large folder. It exists so a crafted request cannot +/// hand `= ANY($2)` an arbitrarily long array. +const MAX_ASKED_RELATIONS: usize = 1_000; + +/// A stored project graph: a deployed version, or one job's snapshot of it. +/// `script_hash` is NULL for an editor buffer's parse, which names no version. +type ProjectVersion = (String, Option, uuid::Uuid); + +async fn dbt_column_lineage( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(pairs): Query>, +) -> JsonResult { + // `None`: pinning to one run is job-scoped and this endpoint is authorized + // as `assets:read`. See `dbt_column_lineage_for`. + let q = ColumnLineageQuery::from_query_pairs(pairs)?; + dbt_column_lineage_for(&authed, &w_id, user_db, q, None).await +} + +/// The column-level lineage the asked-for relations sit in, optionally as one +/// run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF, on the same contract as `asset_graph_for`: +/// `assets:read` always, and the job-read gate for `Some(pinned)`, whose path +/// and hash are then taken from that job's row rather than from the caller. +/// +/// A column trace is transitive and a relation is not owned by one project, so +/// the answer grows a project at a time: resolve who owns the relations reached +/// so far, read their edges, walk, and repeat for the relations that walk newly +/// reached. Every round re-applies the caller's gate to the projects it +/// discovers — a relation being reachable from a project the caller may read +/// says nothing about the project on the other side of it. +pub async fn dbt_column_lineage_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + q: ColumnLineageQuery, + pinned: Option, +) -> JsonResult { + // A column-level view is the shape of what the author WROTE, so it takes the + // model's own gate rather than the relation's. + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(authed, "scripts", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + let (pinned_path, script_hash) = match pinned.as_ref() { + // The job's own version, so a pin cannot name one project's run while + // claiming another's version — including when it names NONE, which is + // the editor buffer. + Some(p) => (Some(p.script_path.as_str()), p.script_hash), + None => (None, q.dbt_script_hash.map(|h| h.0)), + }; + let pinned_job_id = pinned.as_ref().map(|p| p.job_id); + + let seeds: BTreeSet = q.asset_paths.into_iter().collect(); + let mut tx = user_db.begin(authed).await?; + + // Relations whose owners have been asked for, project graphs already read, + // and the edges they yielded. These are what end the loop: a round asks only + // about relations not asked about before and reads only projects not read + // before, so it stops as soon as one of the two runs out. + let mut asked: HashSet = HashSet::new(); + let mut read: HashSet = HashSet::new(); + let mut edges: Vec = Vec::new(); + let mut answer: Vec = Vec::new(); + let mut pending: Vec = seeds.iter().cloned().collect(); + let mut truncated = false; + let mut rounds = 0usize; + + loop { + if pending.is_empty() { + break; + } + // Which project version owns each of these relations, under this + // caller's access. Usually one row per relation; a relation a second + // project declares as a source has two, and each answers for its own + // lineage. + let owners = sqlx::query!( + r#"SELECT DISTINCT n.script_path AS "script_path!", n.script_hash, n.job_id AS "job_id!" + FROM dbt_node n + WHERE n.workspace_id = $1 AND n.asset_path = ANY($2) + -- The run's snapshot, or the deployed graph when that job stored + -- none -- a build pins only if it wrote one. + AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $5) + THEN $5::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END + -- The gate, re-decided for every project the walk reaches. That + -- is what resolving owners in a loop is for: being entitled to + -- one project is not being entitled to the one that declares a + -- relation it hands over. + AND ( $6 + OR n.script_path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE n.script_path = pfx + OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) ) + AND CASE + -- Pinned: which version comes from a job this caller was + -- already granted, so `script` does not decide THAT -- but + -- it still decides whether the project may be read, the + -- same second gate `script_visible` is on the graph. Being + -- entitled to a run is not being entitled to the SQL + -- behind it, and column lineage is that SQL's shape. A + -- version-less row is exempt because it is an editor + -- buffer, which has no `script` row to ask and reaches + -- this only through the parse job that wrote it. + -- + -- One project answers, so a pinned trace never crosses + -- into another: neither does the graph it annotates. + WHEN $4::text IS NOT NULL + THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint + AND ($3::bigint IS NULL OR EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3)) + -- A named version: the deployed one an editor is drawing. + -- `script` is read under RLS, so this is the visibility + -- check as well as the existence one. A hash names one + -- script row, so this arm answers for one project too — + -- and deliberately: a pin says which stored graph is on + -- screen, and another project's live graph is not it. + WHEN $3::bigint IS NOT NULL + THEN n.script_hash = $3 AND EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3) + -- Otherwise the version deployed now: an older one's rows + -- outlive it in `dbt_node` until the sweep, and describe a + -- project that is no longer what runs. `language` narrows + -- it the way the graph's own resolution does, so a path + -- that has since become a script of another kind draws and + -- explains the same version rather than disagreeing. Read + -- under RLS, so a project the caller cannot see resolves + -- to NULL and matches nothing. + ELSE n.script_hash = ( + SELECT sc.hash FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.language = 'dbt' + AND sc.deleted = false AND sc.archived = false + ORDER BY sc.created_at DESC LIMIT 1) + END"#, + w_id, + &pending[..], + script_hash, + pinned_path, + pinned_job_id, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + asked.extend(pending.drain(..)); + + let fresh: Vec = owners + .into_iter() + .map(|o| (o.script_path, o.script_hash, o.job_id)) + .filter(|k| read.insert(k.clone())) + .collect(); + // Nothing left this caller may read and has not read: the component is + // whole, however many relations were still waiting to be asked about. + // Their owners are projects already in hand. + if fresh.is_empty() { + break; + } + // From here a project exists that this answer will not contain, so the + // two stops below are cuts and are reported as such. Deciding it after + // the owners query rather than before is what keeps a big project's + // small component from being called truncated: `pending` alone only says + // a relation has not been ASKED about, not that anything was left out. + rounds += 1; + if rounds > MAX_OWNER_ROUNDS || edges.len() >= EXPANSION_EDGE_BUDGET { + truncated = true; + break; + } + let fresh_paths: Vec = fresh.iter().map(|k| k.0.clone()).collect(); + let fresh_hashes: Vec> = fresh.iter().map(|k| k.1).collect(); + let fresh_jobs: Vec = fresh.iter().map(|k| k.2).collect(); + + // DIRECT kinds only. `scan` -- the column was read to produce the ROW, + // not the value -- reaches every output column of its model, so it is + // most of a project's stored lineage and none of what a trace draws. + // It stays in the table for a later view to ask for. + let rows = sqlx::query!( + r#"SELECT p.asset_path AS "from_path!", e.parent_column AS "from_column!", + c.asset_path AS "to_path!", e.child_column AS "to_column!", + e.lineage_kind AS "kind!" + FROM unnest($2::text[], $3::bigint[], $4::uuid[]) + AS o(script_path, script_hash, job_id) + JOIN dbt_column_edge e ON e.workspace_id = $1 + AND e.script_path = o.script_path + AND e.job_id = o.job_id + -- `=` still, with the NULL-to-NULL case + -- spelled out and gated on the pin: a + -- version-less row's hash is NULL on both + -- sides, which `=` never matches, but + -- `IS NOT DISTINCT FROM` would cost the + -- equality its index bound everywhere else. + AND (e.script_hash = o.script_hash + OR ($5::text IS NOT NULL + AND o.script_hash IS NULL + AND e.script_hash IS NULL)) + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND p.script_hash IS NOT DISTINCT FROM e.script_hash + AND p.job_id = e.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND c.script_hash IS NOT DISTINCT FROM e.script_hash + AND c.job_id = e.job_id + AND c.unique_id = e.child_unique_id + WHERE e.lineage_kind IN ('copy', 'mod') + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL"#, + w_id, + &fresh_paths[..], + &fresh_hashes[..] as &[Option], + &fresh_jobs[..], + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + edges.extend(rows.into_iter().map(|r| DbtColumnLineageEdge { + from_asset_path: r.from_path, + from_column: r.from_column, + to_asset_path: r.to_path, + to_column: r.to_column, + kind: r.kind, + })); + // Two projects can describe one relation, so the same edge can arrive + // twice. Sorted as well as deduplicated: the walk reads the incidence + // lists in this order, so the answer does not depend on which round a + // project was discovered in. + edges.sort(); + edges.dedup(); + + let walked = component(&edges, &seeds); + answer = walked.edges; + truncated = walked.truncated; + if truncated { + break; + } + // The relations the walk newly reached. Their owners are the next round's + // question: this project declares them, and so may another. + pending = answer + .iter() + .flat_map(|e| [&e.from_asset_path, &e.to_asset_path]) + .filter(|p| !asked.contains(*p)) + .cloned() + .collect::>() + .into_iter() + .collect(); + } + tx.commit().await?; + Ok(Json(ColumnLineageResponse { truncated, edges: answer })) +} + +struct WalkedComponent { + edges: Vec, + truncated: bool, +} + +/// The edges of the connected component the asked-for relations sit in, nearest +/// first and at most `MAX_TRACE_EDGES` of them. +/// +/// The canvas lays out the component of the selected relation's columns, so a +/// project's other model families are edges nothing it draws can reach. Walked +/// here rather than in SQL: a recursive CTE has no index to walk, so it rescans +/// the whole edge set once per level — measured at 1.24s against 59ms for the +/// query alone on a 3000-model project, for a walk that is microseconds over a +/// map. Columns are keyed by relation, not by project, which is how the canvas +/// keys them too: two projects describing one relation draw one node. +/// +/// Breadth-first, so the bound cuts the far end of the trace rather than an +/// arbitrary part of it. +fn component(edges: &[DbtColumnLineageEdge], seeds: &BTreeSet) -> WalkedComponent { + let mut incident: HashMap<(&str, &str), Vec> = HashMap::new(); + for (i, e) in edges.iter().enumerate() { + incident + .entry((&e.from_asset_path, &e.from_column)) + .or_default() + .push(i); + incident + .entry((&e.to_asset_path, &e.to_column)) + .or_default() + .push(i); + } + let mut start: Vec<(&str, &str)> = incident + .keys() + .filter(|(path, _)| seeds.contains(*path)) + .copied() + .collect(); + start.sort(); + let mut seen_node: HashSet<(&str, &str)> = start.iter().copied().collect(); + let mut queue: VecDeque<(&str, &str)> = start.into(); + let mut taken = vec![false; edges.len()]; + let mut kept: Vec = Vec::new(); + let mut truncated = false; + 'walk: while let Some(node) = queue.pop_front() { + for &i in incident.get(&node).map(Vec::as_slice).unwrap_or_default() { + if std::mem::replace(&mut taken[i], true) { + continue; + } + if kept.len() == MAX_TRACE_EDGES { + truncated = true; + break 'walk; + } + kept.push(i); + let e = &edges[i]; + let ends = [ + (e.from_asset_path.as_str(), e.from_column.as_str()), + (e.to_asset_path.as_str(), e.to_column.as_str()), + ]; + for end in ends { + if seen_node.insert(end) { + queue.push_back(end); + } + } + } + } + // Back into edge order, so a response does not carry the walk's shape. + kept.sort_unstable(); + WalkedComponent { edges: kept.into_iter().map(|i| edges[i].clone()).collect(), truncated } +} + async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, @@ -1323,7 +1790,7 @@ pub async fn asset_graph_for( n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, - n.columns, n.freshness, + n.columns, n.column_schema, n.freshness, n.raw_code, n.original_file_path, -- Whether the caller may read the project this row describes. -- The query deliberately reaches outside the requested folder @@ -1379,6 +1846,10 @@ pub async fn asset_graph_for( // `ref()` lineage between two models, resolved to the relations they // produce. Joined to `dbt_node` on both key columns because a dbt // `unique_id` is only unique within its project. + // + // Column lineage is NOT here. It is stored per relation and per column, and + // this response is folder-wide and polled by a run page, so it carries only + // what the canvas draws for every node at once. let dbt_edge_rows = sqlx::query!( r#"WITH live AS ( SELECT * FROM ( @@ -1604,6 +2075,7 @@ pub async fn asset_graph_for( description: r.description.clone().filter(|_| source_allowed), data_tests: vec![], columns: r.columns.clone().filter(|_| source_allowed), + column_schema: r.column_schema.clone().filter(|_| source_allowed), freshness: r.freshness.clone().filter(|_| source_allowed), }; // One relation can carry rows from several projects — typically a model diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs index 7f4979a96f..ebeee49ecc 100644 --- a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -8,7 +8,9 @@ //! whole dbt half, and every later fix in this area re-touched one of the two. use sqlx::{Pool, Postgres}; -use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_assets::{ + asset_graph_for, dbt_column_lineage_for, ColumnLineageQuery, GraphQuery, PinnedRun, +}; use windmill_api_auth::ApiAuthed; use windmill_common::db::UserDB; @@ -33,6 +35,7 @@ fn outsider() -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -83,6 +86,34 @@ async fn seed(db: &Pool, job: uuid::Uuid) { .execute(db) .await .unwrap(); + // The relation it reads, and the column edge between them. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders', + 'u/a/wh/analytics/raw_orders', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', + 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); // A test node, for the arguments it carries: `accepted_values` spells out a // column's domain. sqlx::query!( @@ -364,7 +395,25 @@ async fn seed_editor_graph(db: &Pool, job: uuid::Uuid) { r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, raw_code, tags) VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft', - 'u/a/wh/analytics/draft', 'select 3', '{}')"#, + 'u/a/wh/analytics/draft', 'select 3', '{}'), + ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src', + 'u/a/wh/analytics/draft_src', 'select 4', '{}')"#, + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); + // A version-less row's `script_hash` is NULL on both sides of every join and + // every visibility check, and `= NULL` is never true — so the column edges + // need the same NULL arm the node query has, or a buffer parse renders its + // columns and none of their lineage. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", WS, PATH, job @@ -450,3 +499,515 @@ async fn an_editor_graph_renders_only_through_its_own_job(db: Pool) { "nor of a run of the deployed version: {deployed_run}" ); } + +async fn column_lineage_q( + db: &Pool, + authed: &ApiAuthed, + pairs: Vec<(String, String)>, + pinned: Option, +) -> serde_json::Value { + let res = dbt_column_lineage_for( + authed, + WS, + UserDB::new(db.clone()), + ColumnLineageQuery::from_query_pairs(pairs).unwrap(), + pinned, + ) + .await + .unwrap(); + serde_json::to_value(&res.0).unwrap() +} + +async fn column_lineage( + db: &Pool, + authed: &ApiAuthed, + asset_paths: &[&str], + pinned: Option, +) -> serde_json::Value { + let pairs = asset_paths + .iter() + .map(|p| ("asset_path".to_string(), p.to_string())) + .collect(); + column_lineage_q(db, authed, pairs, pinned).await +} + +/// The buffer parse's own lineage, which is the case the versionless rows exist +/// for. Its `script_hash` is NULL on both sides of every join and every +/// visibility check, and `= NULL` is never true — so the versionless arm has to +/// be written for it, or a parse renders its columns and none of their lineage. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_editor_buffers_column_lineage_answers_through_its_job(db: Pool) { + let parse = uuid::Uuid::from_u128(9); + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_editor_graph(&db, parse).await; + let admin = ApiAuthed { is_admin: true, ..outsider() }; + + let pinned = PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], Some(pinned)).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/draft_src", + "from_column": "raw", + "to_asset_path": "u/a/wh/analytics/draft", + "to_column": "clean", + "kind": "mod", + }]), + ); + // Unpinned, the same relation resolves through the deployed version, which + // never heard of the buffer's models. + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], None).await["edges"], + serde_json::json!([]), + "a buffer's lineage is reachable only through the job that parsed it" + ); +} + +/// Being entitled to a RUN is not being entitled to the SQL behind it, and +/// column lineage is that SQL's shape. The pinned graph draws the relations for +/// a share-link viewer and redacts what the author wrote; the lineage is the +/// second, and resolving the version from the job must not be mistaken for +/// deciding that too. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_does_not_hand_over_the_projects_column_lineage(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + + assert_eq!( + column_lineage( + &db, + &outsider(), + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([]), + "the run renders for them, its column-level shape does not" + ); + assert_eq!( + column_lineage( + &db, + &ApiAuthed { is_admin: true, ..outsider() }, + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "while a reader of the project gets it" + ); +} + +/// The deployed version of the same two models, plus a `scan` edge beside the +/// direct one: `seed`'s rows are a run's snapshot, and the unpinned answer is +/// the version's own graph. +async fn seed_deployed_orders(db: &Pool) { + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders', + 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders', + 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); +} + +/// A column-level view is the shape of what the author WROTE, so it takes the +/// script's own gate — the same one that keeps `raw_code` behind access to the +/// project. `scan` says the column was read to produce the ROW rather than the +/// value, so it reaches every output column of its model and is never served. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_takes_the_scripts_gate_and_only_the_direct_kinds(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the direct edge, and not the `scan` one beside it" + ); + assert_eq!( + column_lineage(&db, &outsider(), &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([]), + "and nothing at all for a caller who cannot read the project" + ); +} + +/// One project routinely holds model families that share no column, and the +/// canvas lays out the connected component of the selected relation's columns. +/// Answering with the project's other components sends edges nothing can draw. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_stops_at_the_selected_relations_component(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + // A second family in the same project version, reaching neither of the two + // relations `seed` wired together. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock', + 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily', + 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], Some(pinned())).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the orders family, and not the stock one beside it in the same project" + ); + assert_eq!( + column_lineage( + &db, + &admin, + &["u/a/wh/analytics/stock_daily"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + "and the other way round — reached from the child end, which is upstream" + ); +} + +/// A deployed dbt project in `folder`, declaring `parent`'s relation as a source +/// and deriving `child` from it. `orders → mart → secret_out` is three projects +/// chained through two shared relations. +async fn seed_neighbour_project( + db: &Pool, + folder: &str, + hash: i64, + parent: (&str, &str), + child: (&str, &str), +) { + let path = format!("f/{folder}/proj"); + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, $2, $2, '{}', '{}')", + WS, + folder + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + hash, + path, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4, + 'source', $4, $5, '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6, + 'model', $6, $7, '{}')", + WS, + path, + hash, + parent.0, + parent.1, + child.0, + child.1, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + WS, + path, + hash, + parent.0, + child.0, + ) + .execute(db) + .await + .unwrap(); +} + +/// A trace crosses out of the project it started in, and the gate crosses with +/// it. +/// +/// A relation one project produces is another's source, so the component reaches +/// edges the first project's owner set never named — that is what resolving +/// owners to a fixpoint is for. The other half is that the caller's access has +/// to be re-decided for each project discovered on the way: reaching a relation +/// says nothing about who may read the project on the far side of it. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_crosses_projects_only_where_the_caller_may_read_them(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + seed_neighbour_project( + &db, + "mid", + 43, + ("orders", "u/a/wh/analytics/orders"), + ("mart", "u/a/wh/analytics/mart"), + ) + .await; + seed_neighbour_project( + &db, + "secret", + 44, + ("mart", "u/a/wh/analytics/mart"), + ("secret_out", "u/a/wh/analytics/secret_out"), + ) + .await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let reached = |body: &serde_json::Value| { + body["edges"] + .as_array() + .unwrap() + .iter() + .map(|e| e["to_asset_path"].as_str().unwrap().to_string()) + .collect::>() + }; + + let all = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&all), + [ + "u/a/wh/analytics/mart", + "u/a/wh/analytics/orders", + "u/a/wh/analytics/secret_out" + ] + .map(String::from) + .into(), + "two projects out from the one asked about, not one: {all}" + ); + + // Granted the first two folders and not the third. The edges of the project + // they may read are theirs; the one beyond it is not, even though the + // relation joining them is in the answer. + let partial = ApiAuthed { + folders: vec![ + ("private".to_string(), false, false), + ("mid".to_string(), false, false), + ], + ..outsider() + }; + let some = column_lineage(&db, &partial, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&some), + ["u/a/wh/analytics/mart", "u/a/wh/analytics/orders"] + .map(String::from) + .into(), + "the trace stops where the caller's access does: {some}" + ); + + // The other half of the same gate, and the half that is hand-written SQL + // rather than RLS: a token scoped to one folder reaches the projects in it + // and no others, whatever its grants say. + let scoped = ApiAuthed { + is_admin: true, + scopes: Some(vec!["scripts:read:f/private/*".to_string()]), + ..outsider() + }; + let scoped = column_lineage(&db, &scoped, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&scoped), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "and where its scope does: {scoped}" + ); + + // A version pin answers for that version's project alone — the dbt editor, + // which draws one project as of one deploy. Crossing into `mid` here would + // annotate that canvas with relations it does not draw. + let pinned_version = column_lineage_q( + &db, + &admin, + vec![ + ( + "asset_path".to_string(), + "u/a/wh/analytics/orders".to_string(), + ), + ("dbt_script_hash".to_string(), format!("{:016x}", HASH)), + ], + None, + ) + .await; + assert_eq!( + reached(&pinned_version), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "a version pin does not cross into the project beside it: {pinned_version}" + ); +} + +/// The bound on the answer, and that hitting it is said rather than silently +/// cut: a trace that stops reads exactly like one that ends. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_component_past_the_bound_is_cut_and_says_so(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // One direct edge per column pair, more of them than a trace may carry. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 6000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!(body["edges"].as_array().unwrap().len(), 5000); + assert_eq!(body["truncated"], serde_json::json!(true)); +} + +/// The other side of the same flag: a project big enough to stop the expansion +/// still answers a small component WHOLE, and must not claim it was cut. +/// +/// Reaching the size that stops expansion says nothing on its own — nor does a +/// relation whose owners have not been asked about, since those owners are +/// usually the project already in hand. Only a project this caller may read and +/// this answer does not contain is a cut. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_whole_component_in_a_big_project_is_not_called_cut(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // A second family, two relations and one edge, sharing no column with the + // first — the whole of what a selection on it should return. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock', + 'model', 'stock', 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily', + 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + // And enough unrelated families beside them to pass the expansion budget. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 100000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/stock_daily"], None).await; + assert_eq!( + body["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + ); + assert_eq!(body["truncated"], serde_json::json!(false)); +} diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index bef59aea6c..6e5aa6b1da 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -138,12 +138,28 @@ impl AuthCache { w_id: Option, token: &str, ) -> Option { - let mut opt_job_authed = self.get_opt_job_authed_inner(w_id, token).await?; + let mut opt_job_authed = self.get_opt_job_authed_inner(w_id.clone(), token).await?; // Single source of truth: mirror the resolved job_id onto the authed so // every consumer (require_super_admin, ...) sees that this identity came // from a job's WM_TOKEN, even on an AUTH_CACHE hit whose cached authed // predates this field. opt_job_authed.authed.job_id = opt_job_authed.job_id; + // The workspace's guest switch is enforced here, once, for every guest request + // — not per handler, where each guest-reachable route would have to remember + // it. Uncached, so turning guests off takes effect on the next request of every + // guest session and every token derived from one. + if crate::scopes::has_guest_sentinel(opt_job_authed.authed.scopes.as_deref()) { + let Some(w_id) = w_id else { return None }; + let email = &opt_job_authed.authed.email; + match windmill_common::workspaces::guest_session_stands(&self.db, &w_id, email).await { + Ok(true) => {} + Ok(false) => return None, + Err(e) => { + tracing::error!("guest session check failed for {w_id}: {e:#}"); + return None; + } + } + } Some(opt_job_authed) } @@ -159,6 +175,18 @@ impl AuthCache { if is_no_auth() { return Some(OptJobAuthed { authed: no_auth_admin_authed(), job_id: None }); } + // Reject an oversized guest bearer before the cache key is built from it: the key + // copies and hashes the whole token, so the cap should bound that work too. Log it + // like the other guest refusals, since get_opt_job_authed turns None into a bare 401. + if token.starts_with(windmill_common::guest_jwt::BEARER_PREFIX) + && token.len() > windmill_common::guest_jwt::MAX_GUEST_JWT_LEN + { + tracing::error!( + "guest JWT refused: bearer is longer than {} bytes", + windmill_common::guest_jwt::MAX_GUEST_JWT_LEN + ); + return None; + } let key = ( w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string(), @@ -200,6 +228,111 @@ impl AuthCache { None } } + _ if token.starts_with(windmill_common::guest_jwt::BEARER_PREFIX) => { + // A workspace-less route never accepts a guest JWT: the identity is + // pinned to the workspace its claim names, like a DB guest session. + let Some(w_id) = w_id.as_deref() else { + return None; + }; + // Strip exactly one prefix: `trim_start_matches` would strip repeated prefixes, + // so `jwt_guest_jwt_guest_` would reduce to a valid token that verifies and + // is then cached under the full, non-canonical bearer key. + let jwt = token + .strip_prefix(windmill_common::guest_jwt::BEARER_PREFIX) + .unwrap_or(token); + let claims = + match windmill_common::guest_jwt::verify_for_workspace(&self.db, w_id, jwt) + .await + { + Ok(c) => c, + Err(e) => { + tracing::error!("guest JWT auth error for {w_id}: {e:#}"); + return None; + } + }; + // The workspace switch, the instance switch and the app being in guest + // mode, in one answer (guest_app_admits). The door re-reads the switches + // and the no-account rule per request through the sentinel below + // (guest_session_stands), so turning any of them off stops a cached JWT + // session on its next call. + match windmill_common::workspaces::guest_app_admits( + &self.db, + w_id, + &claims.app_path, + ) + .await + { + Ok(true) => {} + Ok(false) => return None, + Err(e) => { + tracing::error!("guest JWT admit check failed for {w_id}: {e:#}"); + return None; + } + } + // Resolve on the lowercased email: accounts are stored lowercased, so a + // mixed-case claim would otherwise slip past the no-account gate and + // resolve an account holder to a guest, and split the activity rows the + // seat count reads. + let email = claims.email.to_lowercase(); + // A guest is someone with no account at all; an account holder is refused, + // never downgraded (the same rule as the signed-in guest mint). + match windmill_common::users::has_any_account(&self.db, &email).await { + Ok(false) => {} + Ok(true) => return None, + Err(e) => { + tracing::error!("guest JWT account check failed: {e:#}"); + return None; + } + } + // The instance allowance, checked and recorded transactionally. A stranger + // past the cap on a capped instance is refused here; a returning guest + // always passes. Recording an account holder is avoided by the check above. + if !admit_and_record_guest_jwt(&self.db, w_id, &email, &claims.app_path).await { + return None; + } + // guest_session_scopes already carries the sentinel, and it is the whole + // grant; a JWT has no label, so the sentinel is what governs it. It also + // re-checks the path holds no scope metacharacter (verify already did). + let scopes = match crate::scopes::guest_session_scopes(&claims.app_path) { + Ok(s) => Some(s), + Err(e) => { + tracing::error!("guest JWT app_path cannot be scoped for {w_id}: {e:#}"); + return None; + } + }; + // The JWT's own expiry caps a token minted from this session. The auth + // cache entry itself is capped far shorter (GUEST_JWT_CACHE_TTL) so a + // rotated or cleared key stops the session on re-verification, within + // minutes, rather than only at exp (up to 24h away). + let credential_expiry = + chrono::Utc.timestamp_nanos(claims.exp as i64 * 1_000_000_000); + let cache_expiry = credential_expiry.min(chrono::Utc::now() + GUEST_JWT_CACHE_TTL); + let authed = ApiAuthed { + username: email.clone(), + email, + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes, + username_override: None, + username_override_is_token_label: false, + is_session_token: false, + token_prefix: Some(safe_token_prefix(token)), + read_only: false, + job_id: None, + credential_expiry: Some(credential_expiry), + }; + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: cache_expiry, + job_id: None, + }, + ); + Some(OptJobAuthed { authed, job_id: None }) + } _ if token.starts_with("jwt_") => { let jwt_token = token.trim_start_matches("jwt_"); @@ -233,6 +366,7 @@ impl AuthCache { token_prefix: claims.audit_span, read_only: false, job_id: None, + credential_expiry: None, }; // Fail closed: a `job_id` claim that does not parse must reject // the token rather than resolve to `None`, which would clear the @@ -347,6 +481,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } else { tracing::warn!( @@ -400,6 +535,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } else { tracing::warn!( @@ -427,6 +563,8 @@ impl AuthCache { } (_, Some(email), super_admin, scopes, label, read_only) => { let is_session_token = is_session_label(label.as_deref()); + let is_guest_session = + windmill_common::auth::is_guest_session_label(label.as_deref()); let (username_override, username_override_is_token_label) = username_override_from_label(label); if w_id.is_some() { @@ -476,6 +614,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } None if super_admin => { @@ -500,6 +639,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }), Err(e) => { tracing::error!( @@ -509,6 +649,37 @@ impl AuthCache { } } } + // A guest session: IdP-authenticated, member of + // nothing. No `usr` lookup, groups or folders, so + // every ACL denies it and the token's scopes are + // its whole grant. After the superadmin arm, so + // that token is never demoted into this one. + None if is_guest_session => { + // The server-minted label is the grant, never + // the `guest` scope (a user-minted token's + // scopes are whatever the caller typed); the + // sentinel is pinned on here so every guest + // control downstream sees a guest regardless. + let scopes = Some(crate::scopes::with_guest_sentinel( + scopes.unwrap_or_default(), + )); + Some(ApiAuthed { + username: email.clone(), + email, + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes, + username_override, + username_override_is_token_label, + is_session_token, + token_prefix: Some(safe_token_prefix(token)), + read_only, + job_id: None, + credential_expiry: None, + }) + } None => None, } } else { @@ -526,6 +697,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only, job_id: None, + credential_expiry: None, }) } } @@ -564,6 +736,7 @@ impl AuthCache { token_prefix: Some(safe_token_prefix(token)), read_only: false, job_id: None, + credential_expiry: None, }; Some(OptJobAuthed { authed, job_id: None }) } else { @@ -574,6 +747,127 @@ impl AuthCache { } } +/// How long a guest JWT resolves from the auth cache before the arm re-runs (and +/// re-reads the key). A guest JWT is not revocable except by the workspace switch or +/// by rotating the key, so the entry must be short enough that a rotated key bites +/// soon, unlike a normal token whose row can be deleted. Also what makes the +/// day-keyed activity dedupe below reachable across a midnight. +const GUEST_JWT_CACHE_TTL: chrono::Duration = chrono::Duration::minutes(5); + +/// A refused JWT (a stranger past the allowance) is remembered this long so a replayed +/// bearer does not take the instance-wide allowance advisory lock on every request. +/// Short, so a stranger admitted once the window frees is re-checked soon. +const GUEST_JWT_REFUSED_TTL: std::time::Duration = std::time::Duration::from_secs(30); + +lazy_static::lazy_static! { + // One `guest_activity` upsert and one `users.login_guest` audit per email, + // workspace and day: the arm re-runs every GUEST_JWT_CACHE_TTL, and neither the + // seat scan nor the audit trail wants a write each time. LRU-bounded; the day is in + // the key, so a new day writes again. + static ref GUEST_JWT_ACTIVITY_CACHE: Cache = Cache::new(2000); + static ref GUEST_JWT_REFUSED_CACHE: Cache = Cache::new(2000); +} + +/// Admit a JWT guest against the instance allowance and record today's activity, in one +/// transaction so the advisory lock in `guest_admission` spans the count check and the +/// row that changes it. Returns false when the allowance refuses the email or on a DB +/// error, both of which deny the guest. Cached per email, workspace and day: a bearer +/// replayed every request runs this at most once a day, and a refused one is remembered +/// briefly so it does not re-take the allowance lock. `email` is already lowercased. +async fn admit_and_record_guest_jwt(db: &DB, w_id: &str, email: &str, app_path: &str) -> bool { + let cache_key = format!("{email}|{w_id}|{}", chrono::Utc::now().date_naive()); + if GUEST_JWT_ACTIVITY_CACHE.get(&cache_key).is_some() { + return true; + } + if GUEST_JWT_REFUSED_CACHE + .get(&cache_key) + .is_some_and(|at| at.elapsed() < GUEST_JWT_REFUSED_TTL) + { + return false; + } + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("guest JWT tx begin failed for {w_id}: {e:#}"); + return false; + } + }; + // The allowance and the row that changes it, in one transaction: guest_admission + // takes a transaction-scoped advisory lock, so the count check and the insert cannot + // race two strangers past the cap. Only a real allowance refusal is negative-cached; + // a transient DB error denies this request but must not lock the email out for 30s. + match windmill_common::workspaces::guest_admission(&mut *tx, email).await { + Ok(()) => {} + Err(e @ windmill_common::error::Error::PermissionDenied(_)) => { + // The guest hits a bare 401 (the reason must not leak to an unauthenticated caller); + // warn so an admin sees the cap in logs, since it is the actionable signal here. + tracing::warn!("guest JWT refused (guest allowance) for {w_id}: {e:#}"); + GUEST_JWT_REFUSED_CACHE.insert(cache_key, std::time::Instant::now()); + return false; + } + Err(e) => { + tracing::error!("guest JWT allowance check failed for {w_id}: {e:#}"); + return false; + } + } + // The conditional `WHERE NOT jwt_entry` flips the flag only on its false-to-true + // transition, so the upsert returns a row exactly once per email per day: on the + // fresh insert, or on the first JWT after an identity-provider sign-in created + // today's row with `jwt_entry = false`. The audit is gated on that, decided + // atomically by the conflicting tuple, so concurrent first requests (a metered + // instance takes no advisory lock) audit at most once. + let first_jwt = sqlx::query_scalar!( + r#"INSERT INTO guest_activity (email, workspace_id, day, jwt_entry) + VALUES ($1, $2, CURRENT_DATE, true) + ON CONFLICT (email, workspace_id, day) + DO UPDATE SET jwt_entry = true, last_seen_at = now() + WHERE NOT guest_activity.jwt_entry + RETURNING 1 AS "audited!""#, + email, + w_id, + ) + .fetch_optional(&mut *tx) + .await; + let first_jwt = match first_jwt { + Ok(v) => v.is_some(), + Err(e) => { + tracing::error!("recording guest JWT activity for {w_id}: {e:#}"); + return false; + } + }; + if let Err(e) = tx.commit().await { + tracing::error!("guest JWT tx commit failed for {w_id}: {e:#}"); + return false; + } + GUEST_JWT_ACTIVITY_CACHE.insert(cache_key, ()); + // Audit last, best-effort, on its own connection: the EE writer swallows an + // `audit_partitioned` failure but that failing statement still aborts the + // transaction it runs in, so auditing before the commit would let the whole + // activity row roll back while this returned success, admitting an uncounted guest. + if first_jwt { + let author = windmill_common::audit::AuditAuthor { + email: email.to_string(), + username: email.to_string(), + username_override: None, + token_prefix: None, + }; + if let Err(e) = windmill_audit::audit_oss::audit_log( + db, + &author, + "users.login_guest", + windmill_audit::ActionKind::Create, + w_id, + Some(app_path), + Some([("entry", "jwt")].into()), + ) + .await + { + tracing::error!("auditing guest JWT login for {w_id}: {e:#}"); + } + } + true +} + pub(crate) async fn extract_token(parts: &mut Parts, state: &S) -> Option { let auth_header = parts .headers @@ -774,6 +1068,7 @@ fn no_auth_admin_authed() -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -934,10 +1229,17 @@ pub(crate) fn username_override_from_label(label: Option) -> (Option ( - Some(format!("{}{label}", crate::GENERIC_TOKEN_LABEL_PREFIX)), - true, - ), + Some(label) + if label != "ephemeral-script" + && label != "session" + && label != windmill_common::auth::GUEST_SESSION_LABEL + && !label.is_empty() => + { + ( + Some(format!("{}{label}", crate::GENERIC_TOKEN_LABEL_PREFIX)), + true, + ) + } _ => (None, false), } } diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index f2392839ba..bb1586db68 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -78,6 +78,11 @@ pub struct ApiAuthed { /// member can point at a superadmin, so it must never be trusted as a global /// superadmin (`require_super_admin`), GHSA-hfh4-cx4h-3fcr. pub job_id: Option, + /// When this credential itself expires, if it carries its own expiry rather than a + /// token row. Set for a guest JWT (its `exp`): a token minted from it is capped at + /// this, since the JWT's expiry is a guest's only revocation and there is no row to + /// look the limit up in. `None` for every credential whose limit lives in `token`. + pub credential_expiry: Option>, } impl ApiAuthed { @@ -165,6 +170,7 @@ impl From for ApiAuthed { token_prefix: value.token_prefix, read_only: false, job_id: None, + credential_expiry: None, } } } @@ -1047,6 +1053,10 @@ pub async fn fetch_api_authed_from_permissioned_as( db: &DB, username_override: Option, ) -> error::Result { + // Keyed by the supplied address, so an entry built for a principal's previous holder is reused + // while that address is still supplied, until its 120s expiry: a cached dispatch address is + // evicted sooner, an app's stored one (a username deleted then reused) may not be. Accepted; + // the rebuild after expiry is the current holder's. let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); let mut api_authed = match API_AUTHED_CACHE.get(&key) { @@ -1062,7 +1072,10 @@ pub async fn fetch_api_authed_from_permissioned_as( let api_authed = ApiAuthed { username: authed.username, - email, + // The resolved one, not the address we were handed: that is the point of + // `fetch_authed_from_permissioned_as` validating it against the principal's live + // binding, and this value goes on to the job row, `job_perms` and the JWT. + email: authed.email, is_admin: authed.is_admin, is_operator: authed.is_operator, groups: authed.groups, @@ -1074,6 +1087,7 @@ pub async fn fetch_api_authed_from_permissioned_as( token_prefix: authed.token_prefix, read_only: false, job_id: None, + credential_expiry: None, }; API_AUTHED_CACHE.insert( @@ -1127,6 +1141,9 @@ impl NewToken { /// [`ensure_scopes_within_caller`] first (internal narrowing mints intentionally /// skip it, since their scopes derive from the action being authorized, not the /// caller's token). +/// +/// A token the system mints for itself with an `expiration` needs a label reserved in +/// `windmill_common::auth::is_user_token`, or its expiry alerts its owner (docs/auth-surface.md). pub async fn create_token_internal( tx: &mut sqlx::PgConnection, db: &DB, @@ -1232,6 +1249,10 @@ pub async fn register_token_expiry_notification( if !windmill_common::auth::is_user_token(label) { return; } + let warning_days = windmill_common::auth::TOKEN_EXPIRY_WARNING_DAYS; + if expiration <= chrono::Utc::now() + chrono::Duration::days(warning_days.into()) { + return; + } if let Err(e) = sqlx::query!( "INSERT INTO token_expiry_notification (token_hash, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", token_hash, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index d99d955de9..29ee0fe36a 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -288,7 +288,6 @@ pub enum ScopeDomain { Configs, OAuth, AI, - AiSkills, AiEvals, // AI agent eval datasets Indexer, @@ -349,7 +348,6 @@ impl ScopeDomain { Self::Configs => "configs", Self::OAuth => "oauth", Self::AI => "ai", - Self::AiSkills => "ai_skills", Self::AiEvals => "ai_evals", Self::Capture => "capture", Self::Drafts => "drafts", @@ -405,7 +403,6 @@ impl ScopeDomain { "configs" => Some(Self::Configs), "oauth" => Some(Self::OAuth), "ai" => Some(Self::AI), - "ai_skills" => Some(Self::AiSkills), "ai_evals" => Some(Self::AiEvals), "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), @@ -503,6 +500,24 @@ pub fn check_route_access( } } + // A guest session carries the same broad read scopes as an embed token and for + // the same handful of routes, so it gets the same default-deny. + if has_guest_sentinel(Some(token_scopes)) { + if let Some(suffix) = route_suffix.as_deref() { + if guest_route_denied(required_domain, suffix) { + return Err(Error::PermissionDenied(format!( + "a guest session cannot access {route_path}" + ))); + } + // Same rationale as the embed branch: re-running a component supersedes + // its in-flight run, and `cancel_job_api` confines this to the caller's + // own jobs. + if suffix.starts_with("jobs_u/queue/cancel/") { + return Ok(()); + } + } + } + // Each declared scope must grant what its prompt said and no more: // `jobs:run` only deployed runnables, `users:read` only the viewer's identity. if has_raw_app_sdk_sentinel(Some(token_scopes)) { @@ -756,6 +771,55 @@ pub fn has_app_embed_sentinel(scopes: Option<&[String]>) -> bool { scopes.is_some_and(|s| s.iter().any(|x| x == APP_EMBED_SENTINEL)) } +/// Sentinel in a guest session token: someone the identity provider authenticated +/// who is a member of no workspace. Grants nothing itself — it only confines the +/// session to the app surface, the same way `app_embed` does. What makes a session a +/// guest at all is the server-minted label +/// [`windmill_common::auth::GUEST_SESSION_LABEL`]; a forged sentinel here can only +/// narrow its own token. +pub const GUEST_SENTINEL: &str = "guest"; + +/// True if a token is a guest session, whose scopes are its entire grant: it has no ACL +/// of its own, so every ACL check denies it unaided. +pub fn has_guest_sentinel(scopes: Option<&[String]>) -> bool { + scopes.is_some_and(|s| s.iter().any(|x| x == GUEST_SENTINEL)) +} + +/// `scopes` with the guest sentinel present exactly once. +pub fn with_guest_sentinel(mut scopes: Vec) -> Vec { + if !scopes.iter().any(|x| x == GUEST_SENTINEL) { + scopes.push(GUEST_SENTINEL.to_string()); + } + scopes +} + +/// Scopes a guest session carries. The broad-looking reads are narrowed to a route +/// allowlist by the sentinel (`guest_route_denied`), plus the two path-scoped app +/// grants. A guest has no `usr` row, so this list is the whole of what it can do. The +/// single source both the mint (a signed-in guest) and the JWT auth arm build from. +/// +/// The sentinel here only narrows. A signed-in guest is made one by the server-minted +/// label; a JWT guest has no label, so for it the sentinel is what governs. +pub fn guest_session_scopes(app_path: &str) -> windmill_common::error::Result> { + // The path is spliced into a scope, whose grammar reserves `:`, `,`, `*` and a leading + // `/`; app paths may otherwise carry spaces and `@`, so guard only those reserved chars. + if !windmill_common::auth::is_scope_literal_path(app_path) { + return Err(windmill_common::error::Error::BadRequest(format!( + "app path {app_path} is empty or cannot be scoped: `:`, `,` and `*` are reserved \ + in scopes, and a leading `/` never matches a route" + ))); + } + Ok(vec![ + GUEST_SENTINEL.to_string(), + "jobs:read".to_string(), + "resources:run".to_string(), + "users:read".to_string(), + "folders:read".to_string(), + format!("apps:read:{app_path}"), + format!("apps:run:{app_path}"), + ]) +} + /// Sentinel in raw-app SDK tokens. Grants nothing; `check_route_access` uses it /// to narrow the declared scopes to what the viewer's prompt promised. pub const RAW_APP_SDK_SENTINEL: &str = "raw_app_sdk"; @@ -818,6 +882,19 @@ fn app_embed_apps_route_allowed(suffix: &str) -> bool { suffix.starts_with("apps/get/p/") || suffix.starts_with("apps_u/") } +/// Routes a guest session is denied: the app-embed allowlist, plus the embed-token +/// mint. A guest session is the *embedder* — the viewer's own browser rendering the +/// app page — not the app's own JS, and the page mints the iframe's token from it. +/// +/// Everything else stays default-denied, so a guest reaches the app it was let in +/// for and nothing around it. +fn guest_route_denied(domain: ScopeDomain, suffix: &str) -> bool { + if domain == ScopeDomain::Apps && suffix.starts_with("apps_u/embed_token") { + return false; + } + app_embed_route_denied(domain, suffix) +} + /// Job routes a running app uses (the by-id poll/cancel surface driven by the /// frontend JobLoader). Everything else in the jobs domain — enumeration, counts, /// exports, and the `job_signature`/`resume_urls` capability-minting routes — is @@ -996,12 +1073,15 @@ fn scope_grants_access( /// the caller's own row; `email` and `allowed_domain_auto_invite` are derived from the /// token itself and touch no table. /// -/// `settings/global/automate_username_creation` is the one instance setting on the list. -/// `get_global_setting` exempts a handful of keys from its own super-admin gate, that one -/// among them, so the boolean is already readable by every authenticated user; it is here -/// because the CLI reads it before creating a user during a git-sync push, which runs as a -/// job. The other ungated keys have no such caller, so they stay confined — being ungated -/// earns a key nothing on its own. +/// Three instance settings are on the list. `get_global_setting` exempts a handful of keys +/// from its own super-admin gate, these among them, so each is already readable by every +/// authenticated user; each is here because the CLI reads it from a job: +/// `automate_username_creation` before creating a user during a git-sync push, `uid` and +/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other +/// ungated keys have no such caller, so they stay confined — being ungated earns a key +/// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin` +/// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub, +/// stays out of a job's reach whatever this list says. /// /// Deliberately absent, as each crosses that line: `users/list_invites` (returns the /// workspace ids the identity was invited to), `users/tokens/list` (credential metadata @@ -1019,6 +1099,8 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool { | "/api/users/tutorial_progress" | "/api/workspaces/allowed_domain_auto_invite" | "/api/settings/global/automate_username_creation" + | "/api/settings/global/uid" + | "/api/settings/global/hub_base_url" | "/api/docs/search" | "/api/docs/page" | "/api/integrations/hub/list" @@ -1202,12 +1284,6 @@ mod tests { assert_eq!(domain, ScopeDomain::FlowConversations); assert_eq!(kind, None); assert_eq!(route_suffix, Some("flow_conversations/list".to_string())); - - let (domain, kind, route_suffix) = - extract_domain_from_route("/api/w/test_workspace/ai_skills/list").unwrap(); - assert_eq!(domain, ScopeDomain::AiSkills); - assert_eq!(kind, None); - assert_eq!(route_suffix, Some("ai_skills/list".to_string())); } #[test] @@ -1368,11 +1444,6 @@ mod tests { ScopeDomain::from_str("flow_conversations"), Some(ScopeDomain::FlowConversations) ); - assert_eq!( - ScopeDomain::from_str("ai_skills"), - Some(ScopeDomain::AiSkills) - ); - // Test canonical string conversion assert_eq!(ScopeDomain::Acls.as_str(), "acls"); assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); @@ -1381,41 +1452,6 @@ mod tests { ScopeDomain::FlowConversations.as_str(), "flow_conversations" ); - assert_eq!(ScopeDomain::AiSkills.as_str(), "ai_skills"); - } - - #[test] - fn test_ai_skills_scope_access() { - let read_scopes = vec!["ai_skills:read".to_string()]; - assert!( - check_route_access(&read_scopes, "/api/w/test_workspace/ai_skills/list", "GET").is_ok() - ); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/get/foo", - "GET" - ) - .is_ok()); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_err()); - - let write_scopes = vec!["ai_skills:write".to_string()]; - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_ok()); - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/delete/foo", - "DELETE" - ) - .is_ok()); } #[test] diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7ad99996d5..7598021f00 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -59,14 +59,116 @@ struct Config { config: serde_json::Value, } +/// Credential-bearing fields across the `ObjectSettings` variants, which are flattened into a +/// single object by the `type` tag. +const OBJECT_STORE_SECRET_KEYS: &[&str] = + &["access_key", "secret_key", "accessKey", "serviceAccountKey"]; + +/// What an obfuscated read shows a caller who may not see the real credential. +const OBJECT_STORE_SECRET_MASK: &str = "*****"; + +/// Blank the secrets in one worker-group config, in place. +/// +/// Worker-group configs are instance-global and expose `env_vars_static` and the bucket +/// credentials of `object_store_cache_config`; a job token (capped at workspace admin) gets this +/// view even when its identity is a superadmin, as does a devops user who is not an instance +/// admin. See `is_instance_admin` (GHSA-hfh4-cx4h-3fcr). Every route that returns a worker-group +/// config must go through here — a single unobfuscated read hands over the whole bucket. +fn obfuscate_worker_config(config: &mut serde_json::Value) { + let Some(config) = config.as_object_mut() else { + return; + }; + if let Some(env_vars) = config + .get_mut("env_vars_static") + .and_then(|v| v.as_object_mut()) + { + for (_, value) in env_vars.iter_mut() { + // the value is a string, so to_string() it and take -2 to drop the quotes + *value = serde_json::json!("*".repeat(value.to_string().len().saturating_sub(2))); + } + } + if let Some(store) = config + .get_mut("object_store_cache_config") + .and_then(|v| v.as_object_mut()) + { + for key in OBJECT_STORE_SECRET_KEYS { + if let Some(secret) = store.get_mut(*key) { + *secret = serde_json::json!(OBJECT_STORE_SECRET_MASK); + } + } + } +} + +/// Put back the credentials behind [`OBJECT_STORE_SECRET_MASK`]. A devops user who is not an +/// instance admin edits the group from the obfuscated view, so a plain save would otherwise +/// store the mask as the secret and take the group's dependency cache offline — silently, since +/// a worker that cannot build its override just falls back to caching on local disk. +async fn restore_masked_object_store_secrets( + db: &DB, + name: &str, + config: &mut serde_json::Value, +) -> error::Result<()> { + if !has_masked_object_store_secret(config) { + return Ok(()); + } + let stored = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(db) + .await? + .map(|c| c.config); + restore_object_store_secrets(config, stored.as_ref()); + Ok(()) +} + +fn has_masked_object_store_secret(config: &serde_json::Value) -> bool { + let Some(store) = config.get("object_store_cache_config") else { + return false; + }; + OBJECT_STORE_SECRET_KEYS + .iter() + .any(|k| store.get(k).and_then(|v| v.as_str()) == Some(OBJECT_STORE_SECRET_MASK)) +} + +/// The half of [`restore_masked_object_store_secrets`] after the read. +fn restore_object_store_secrets( + config: &mut serde_json::Value, + stored: Option<&serde_json::Value>, +) { + let stored = stored + .and_then(|c| c.get("object_store_cache_config")) + .and_then(|v| v.as_object()) + .cloned(); + let Some(store) = config + .get_mut("object_store_cache_config") + .and_then(|v| v.as_object_mut()) + else { + return; + }; + for key in OBJECT_STORE_SECRET_KEYS { + if store.get(*key).and_then(|v| v.as_str()) != Some(OBJECT_STORE_SECRET_MASK) { + continue; + } + match stored.as_ref().and_then(|s| s.get(*key)) { + Some(secret) => store.insert(key.to_string(), secret.clone()), + // Nothing to put back: drop the mask rather than store it. + None => store.remove(*key), + }; + } +} + async fn list_worker_groups( authed: ApiAuthed, Extension(db): Extension, ) -> error::JsonResult> { - let mut configs_raw = - sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'") - .fetch_all(&db) - .await?; + let mut configs_raw = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name LIKE 'worker__%'" + ) + .fetch_all(&db) + .await?; // Remove the 'worker__' prefix from all config names for config in configs_raw.iter_mut() { if let Some(name) = &config.name { @@ -75,44 +177,12 @@ async fn list_worker_groups( } } } - // Worker-group configs are instance-global and expose env_vars_static (may hold - // secrets); a job token (capped at workspace admin) gets the obfuscated view even - // when its identity is a superadmin. See is_instance_admin (GHSA-hfh4-cx4h-3fcr). - let configs = if !is_instance_admin(&authed) { - let mut obfuscated_configs: Vec = vec![]; - for config in configs_raw { - let config_value_opt = config.config.as_object().map(|obj| obj.to_owned()); - if let Some(mut config_value) = config_value_opt { - if let Some(env_var_map) = config_value - .get("env_vars_static") - .map(|obj| obj.as_object()) - .flatten() - { - let mut new_env_var_map: serde_json::Map = - serde_json::Map::new(); - for (key, value) in env_var_map { - new_env_var_map.insert( - key.to_owned(), - // we know the value is a string here, so we to_string() it and take -2 to remove the quotes - serde_json::json!("*".repeat(value.to_string().len() - 2)), - ); - } - config_value.insert( - "env_vars_static".to_string(), - serde_json::Value::Object(new_env_var_map), - ); - } - obfuscated_configs.push(Config { - name: config.name, - config: serde_json::Value::Object(config_value), - }) - } + if !is_instance_admin(&authed) { + for config in configs_raw.iter_mut() { + obfuscate_worker_config(&mut config.config); } - obfuscated_configs - } else { - configs_raw - }; - Ok(Json(configs)) + } + Ok(Json(configs_raw)) } async fn get_config( @@ -122,10 +192,20 @@ async fn get_config( ) -> error::JsonResult> { require_devops_role(&db, &authed).await?; - let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name) - .fetch_optional(&db) - .await? - .map(|c| c.config); + let mut config = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(&db) + .await? + .map(|c| c.config); + + if !is_instance_admin(&authed) { + if let Some(config) = config.as_mut() { + obfuscate_worker_config(config); + } + } Ok(Json(config)) } @@ -134,10 +214,14 @@ async fn update_config( Path(name): Path, Extension(db): Extension, authed: ApiAuthed, - Json(config): Json, + Json(mut config): Json, ) -> error::Result { require_devops_role(&db, &authed).await?; + if name.starts_with("worker__") { + restore_masked_object_store_secrets(&db, &name, &mut config).await?; + } + #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode @@ -321,9 +405,14 @@ async fn list_configs( Extension(db): Extension, ) -> error::JsonResult> { require_devops_role(&db, &authed).await?; - let configs = sqlx::query_as!(Config, "SELECT name, config FROM config") + let mut configs = sqlx::query_as!(Config, "SELECT name, config FROM config") .fetch_all(&db) .await?; + if !is_instance_admin(&authed) { + for config in configs.iter_mut() { + obfuscate_worker_config(&mut config.config); + } + } Ok(Json(configs)) } @@ -414,3 +503,49 @@ async fn list_all_dedicated_with_deps( Ok(Json(result)) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The mask an obfuscated read hands out must never be storable as the credential itself: + /// a devops user who is not an instance admin edits the group from that view, and a worker + /// that cannot build its override degrades to a local-only cache without failing a job, so + /// the breakage would go unnoticed. + #[test] + fn masked_secrets_survive_a_save_from_the_obfuscated_view() { + let stored = serde_json::json!({ + "object_store_cache_config": { + "type": "S3", "bucket": "cache", "access_key": "AKIA", "secret_key": "s3cr3t" + }, + "env_vars_static": { "TOKEN": "hunter2" }, + }); + + let mut shown = stored.clone(); + obfuscate_worker_config(&mut shown); + let store = &shown["object_store_cache_config"]; + assert_eq!(store["secret_key"], OBJECT_STORE_SECRET_MASK); + assert_eq!(store["access_key"], OBJECT_STORE_SECRET_MASK); + assert_eq!(store["bucket"], "cache"); + assert_ne!(shown["env_vars_static"]["TOKEN"], "hunter2"); + + let mut saved = shown.clone(); + saved["object_store_cache_config"]["bucket"] = serde_json::json!("other"); + restore_object_store_secrets(&mut saved, Some(&stored)); + let store = &saved["object_store_cache_config"]; + assert_eq!(store["secret_key"], "s3cr3t"); + assert_eq!(store["access_key"], "AKIA"); + assert_eq!(store["bucket"], "other"); + } + + #[test] + fn a_mask_with_nothing_behind_it_is_dropped_rather_than_stored() { + let mut saved = serde_json::json!({ + "object_store_cache_config": { "type": "S3", "secret_key": OBJECT_STORE_SECRET_MASK } + }); + restore_object_store_secrets(&mut saved, None); + assert!(saved["object_store_cache_config"] + .get("secret_key") + .is_none()); + } +} diff --git a/backend/windmill-api-debug/src/lib.rs b/backend/windmill-api-debug/src/lib.rs index 7b2f569821..ab89df988d 100644 --- a/backend/windmill-api-debug/src/lib.rs +++ b/backend/windmill-api-debug/src/lib.rs @@ -468,6 +468,9 @@ async fn sign_debug_request( // Parse the language let script_lang: ScriptLang = request.language.parse().unwrap_or(ScriptLang::Bun); + // Taken from the parsed language, not the request's string: the telemetry key vocabulary has + // to stay the closed set of languages rather than whatever a caller sent. + let lang_key = script_lang.as_str(); // Hash the code (we don't include full code in JWT to keep it small) let mut hasher = Sha256::new(); @@ -578,6 +581,8 @@ async fn sign_debug_request( tx.commit().await?; + windmill_common::feature_usage::log_feature_usage("debugger", "session", lang_key); + Ok(Json(SignedDebugPayload { token, code: request.code, diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 35816dfaa8..1de99fb44f 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -418,7 +418,7 @@ impl EmbeddingsDb { let hub_resource_types = response.json::>().await?; let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name",) + sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name",) .fetch_all(pg_db) .await?; diff --git a/backend/windmill-api-flow-conversations/src/lib.rs b/backend/windmill-api-flow-conversations/src/lib.rs index e85af5b83b..0ec295c23f 100644 --- a/backend/windmill-api-flow-conversations/src/lib.rs +++ b/backend/windmill-api-flow-conversations/src/lib.rs @@ -1,6 +1,6 @@ use axum::{ extract::{Path, Query}, - routing::{delete, get}, + routing::{delete, get, post}, Extension, Json, Router, }; use chrono::{DateTime, Utc}; @@ -15,13 +15,14 @@ use windmill_common::{ db::{UserDB, DB}, error::{JsonResult, Result}, flow_conversations::MessageType, - utils::{not_found_if_none, paginate, Pagination}, + utils::{not_found_if_none, paginate, truncate_with_ellipsis, Pagination}, }; pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_conversations)) .route("/delete/{conversation_id}", delete(delete_conversation)) + .route("/update/{conversation_id}", post(update_conversation)) .route("/{conversation_id}/messages", get(list_messages)) } @@ -36,11 +37,37 @@ pub struct FlowConversationMessage { pub created_seq: i64, pub step_name: Option, pub success: bool, + /// On a tool row, the arguments the model wrote. For a Windmill tool these exclude the + /// inputs its step wires in. Null for a web search, whose query the provider does not + /// return. + pub tool_arguments: Option, + /// On a tool row, the text the model got back, or what the call failed with; a web + /// search's citations. + pub tool_result: Option, + /// On an answer, the thinking that produced it; on a tool row, the thinking that led to + /// the call. The agent job keeps the turn's thinking as one string. + pub reasoning: Option, + /// The files a user message carried, as object-storage references + /// (`[{input, s3, storage?, filename?}]`). + pub attachments: Option, +} + +/// Which conversations a listing holds. A test chat was started from the editor's test +/// panel; a deployed one from the flow itself. +#[derive(Deserialize, Default, Clone, Copy)] +#[serde(rename_all = "lowercase")] +pub enum ConversationKind { + Test, + /// The default: a deployed flow's chat should not surface someone's trial runs. + #[default] + Deployed, + All, } #[derive(Deserialize)] pub struct ListConversationsQuery { pub flow_path: Option, + pub kind: Option, } #[derive(Deserialize)] @@ -67,6 +94,7 @@ async fn list_conversations( "created_at", "updated_at", "created_by", + "is_test", ]) .and_where_eq("workspace_id", "?".bind(&w_id)); @@ -74,6 +102,16 @@ async fn list_conversations( sqlb.and_where_eq("flow_path", "?".bind(flow_path)); } + match query.kind.unwrap_or_default() { + ConversationKind::Test => { + sqlb.and_where_eq("is_test", "true"); + } + ConversationKind::Deployed => { + sqlb.and_where_eq("is_test", "false"); + } + ConversationKind::All => {} + } + sqlb.order_by("updated_at", true) .limit(per_page as i64) .offset(offset as i64); @@ -101,7 +139,7 @@ async fn delete_conversation( // Verify the conversation exists and belongs to the user let conversation = sqlx::query_as!( FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test FROM flow_conversation WHERE id = $1 AND workspace_id = $2", conversation_id, @@ -148,6 +186,50 @@ async fn delete_conversation( Ok(format!("Conversation {} deleted", conversation_id)) } +#[derive(Deserialize)] +pub struct UpdateConversation { + pub title: String, +} + +async fn update_conversation( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, conversation_id)): Path<(String, Uuid)>, + Json(update): Json, +) -> Result { + // Postgres refuses a NUL in a text column, so it must not reach the query as a 500. + if update.title.contains('\0') { + return Err(windmill_common::error::Error::BadRequest( + "title cannot contain a NUL character".to_string(), + )); + } + // The column is VARCHAR(255) and the helper appends an ellipsis to what it cuts, so the + // bound it takes is three short of the column's. A longer title would otherwise reach + // Postgres as a 22001 and come back a 500. + let title = truncate_with_ellipsis(update.title.trim(), 252); + + let mut tx = user_db.clone().begin(&authed).await?; + + // `updated_at` is kept: the list is ordered by it, and a rename must not move the + // chat to the top the way a new turn does. + let updated = sqlx::query_scalar!( + "UPDATE flow_conversation SET title = $1, updated_at = updated_at + WHERE id = $2 AND workspace_id = $3 + RETURNING id", + title, + conversation_id, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(updated, "Conversation", conversation_id.to_string())?; + + tx.commit().await?; + + Ok(format!("Conversation {} updated", conversation_id)) +} + async fn list_messages( authed: ApiAuthed, Extension(user_db): Extension, @@ -178,7 +260,7 @@ async fn list_messages( let messages = if let Some(after_seq) = query.after_seq { sqlx::query_as!( FlowConversationMessage, - r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success + r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning, attachments FROM flow_conversation_message WHERE conversation_id = $1 AND created_seq > $2 @@ -195,9 +277,9 @@ async fn list_messages( // Fetch messages for this conversation, oldest first, but reverse the order of the messages for easy rendering on the frontend sqlx::query_as!( FlowConversationMessage, - r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success + r#"SELECT id, conversation_id, message_type as "message_type: MessageType", content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning, attachments FROM ( - SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success + SELECT id, conversation_id, message_type, content, job_id, created_at, created_seq, step_name, success, tool_arguments, tool_result, reasoning, attachments FROM flow_conversation_message WHERE conversation_id = $1 ORDER BY created_seq DESC diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 6353cd429a..c7df69879e 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -51,7 +51,10 @@ use windmill_common::{ jobs::JobPayload, schedule::Schedule, triggers::MovedNativeTrigger, - utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, + utils::{ + http_get_from_hub, not_found_if_none, paginate, paginate_optional, Pagination, + RunnableKind, StripPath, + }, }; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -77,6 +80,10 @@ pub fn workspaced_service() -> Router { "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}", get(list_paths_from_workspace_runnable), ) + .route( + "/list_paths_linking_agent/{*path}", + get(list_paths_linking_agent), + ) .route("/history_update/v/{version}", post(update_flow_history)) .route("/get/v/{version}", get(get_flow_version_by_id)) .route("/get/v/{version}/p/{*path}", get(get_flow_version)) @@ -150,6 +157,7 @@ async fn list_flows( "favorite.path IS NOT NULL as starred", "ws_error_handler_muted", "o.labels", + "(o.value->>'chat_input_enabled')::bool as chat_input_enabled", "draft.email IS NOT NULL as is_draft", // Per-path draft owners as a JSON array; see scripts.rs for the rationale // (non-member superadmin identity fallback via `password`, legacy NULL-email row). @@ -157,7 +165,8 @@ async fn list_flows( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() @@ -296,6 +305,10 @@ async fn list_flows( ws_error_handler_muted: None, deployment_msg: None, labels: None, + chat_input_enabled: v + .get("value") + .and_then(|fv| fv.get("chat_input_enabled")) + .and_then(|b| b.as_bool()), // No deployed row to inherit folder labels from. inherited_labels: None, is_draft: true, @@ -507,7 +520,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -520,7 +533,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -533,6 +546,30 @@ async fn list_paths_from_workspace_runnable( Ok(Json(runnables)) } +/// Flows with a step linked to the `ai_agent` resource at `path`, as of their last deploy. +async fn list_paths_linking_agent( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:agent/{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let flows = sqlx::query_scalar!( + r#"SELECT DISTINCT f.path + FROM workspace_runnable_dependencies wru + JOIN flow f + ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id + WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2"#, + path, + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(flows)) +} + async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { #[cfg(not(feature = "enterprise"))] if new_flow.ws_error_handler_muted.is_some_and(|val| val) { @@ -713,6 +750,14 @@ async fn create_flow( .execute(&mut *tx) .await?; } + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::Flow], + &nf.path, + None, + ) + .await?; audit_log( &mut *tx, @@ -826,6 +871,29 @@ async fn create_flow( WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, ); + // Trigger CI tests for items that reference this flow + { + let db2 = db.clone(); + let w_id2 = w_id.clone(); + let flow_path2 = nf.path.clone(); + let email2 = authed.email.clone(); + let username2 = authed.username.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_dep_map::ci_tests::trigger_ci_tests_for_item( + &db2, + &w_id2, + &flow_path2, + "flow", + &email2, + &username2, + ) + .await + { + tracing::error!(%e, "error triggering CI tests after flow creation"); + } + }); + } + Ok((StatusCode::CREATED, nf.path.to_string())) } @@ -858,25 +926,36 @@ pub struct FlowVersion { pub created_at: chrono::DateTime, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + /// Who deployed this version — the diff's version picker names them so a reader + /// can tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, } async fn get_flow_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("flows:read:{}", path))?; + // Unasked-for, this listing stays whole: the history panels, the restart picker and + // the CLI all read it without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; let flows = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version - WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 - ORDER BY flow_version.created_at DESC", + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC + LIMIT $3 OFFSET $4", path, - w_id + w_id, + per_page, + offset, ) .fetch_all(&mut *tx) .await?; @@ -896,7 +975,7 @@ async fn get_latest_version( let version = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 ORDER BY flow_version.created_at DESC", @@ -921,7 +1000,7 @@ async fn derived_on_behalf_of_email( let Some(permissioned_as) = flow.on_behalf_of.as_deref() else { return Ok(None); }; - // Uncached, for the reason given on `prefetch_cached_script`: this pair is round-tripped. + // Uncached: this pair is round-tripped by the client and stored again on redeploy. Ok(Some( windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) .await?, @@ -1381,6 +1460,20 @@ async fn update_flow( .await?; } + if is_new_path { + // Everything left at the old path is a draft this deploy didn't consume + // — teammates' rows, and the deployer's own when the caller asked us to + // keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::Flow], + flow_path, + &nf.path, + ) + .await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-groups/Cargo.toml b/backend/windmill-api-groups/Cargo.toml index 856eaacc0a..35443008f0 100644 --- a/backend/windmill-api-groups/Cargo.toml +++ b/backend/windmill-api-groups/Cargo.toml @@ -29,4 +29,5 @@ serde.workspace = true serde_json.workspace = true sql-builder.workspace = true sqlx.workspace = true +tracing.workspace = true uuid.workspace = true diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index 646f764ec6..c52b8a6316 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -800,6 +800,14 @@ async fn delete_folder( not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; + // See the same call in `delete_group`: a freed name must not stay in a tenant list. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("f/{name}"), + ) + .await?; + let del = sqlx::query_scalar!( "DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1", name, diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 5a7af05cbe..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -183,9 +183,9 @@ async fn add_granular_acl( if kind == "folder" { let change_type = if write.unwrap_or(false) { - "grant_read" - } else { "grant_write" + } else { + "grant_read" }; crate::folders::log_folder_permission_change( &mut *tx, @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index c3719cd608..3422b25200 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -22,7 +22,10 @@ use windmill_common::{ error::{Error, JsonResult, Result}, utils::{not_found_if_none, paginate, Pagination}, }; -use windmill_common::{db::UserDB, users::username_to_permissioned_as}; +use windmill_common::{ + db::UserDB, + users::{username_to_permissioned_as, usr_accepts_email}, +}; use serde::{Deserialize, Serialize}; use sqlx::{query_scalar, FromRow, Postgres, Transaction}; @@ -797,6 +800,15 @@ async fn delete_group( } not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + // A tenant list names a principal, so a freed name must not linger in one: a later group + // reusing it would silently inherit the data table access this one had. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("g/{name}"), + ) + .await?; + sqlx::query!( "DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2", name, @@ -975,6 +987,15 @@ async fn add_user_igroup( ) -> Result { require_super_admin(&db, &authed).await?; + // `email_to_igroup` has no shape constraint of its own; `usr`, which the member is + // promoted into on reconcile, has `proper_email`, and a value failing it there would + // roll back every member of the group. + if !usr_accepts_email(&db, &email).await? { + return Err(Error::BadRequest(format!( + "'{email}' is not a valid email address" + ))); + } + let mut tx: Transaction<'_, Postgres> = db.begin().await?; // FOR UPDATE: the group row is the group-level mutex, taken before the workspace @@ -1427,6 +1448,16 @@ async fn overwrite_igroups( if let Some(emails) = &igroup.emails { for email in emails.iter() { + // An export can carry a member the source instance stored before ingest + // validated member values; it is dropped rather than failing the import. + if !usr_accepts_email(&mut *tx, email).await? { + tracing::warn!( + "Skipping member '{}' of imported instance group '{}': not an email address", + email, + igroup.name + ); + continue; + } sqlx::query!( "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", email, diff --git a/backend/windmill-api-integration-tests/tests/datatable_acl.rs b/backend/windmill-api-integration-tests/tests/datatable_acl.rs new file mode 100644 index 0000000000..4310830ae2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/datatable_acl.rs @@ -0,0 +1,108 @@ +//! Who may read and change a data table's grants and owners. On the Enterprise Edition: its +//! administrators, from the workspace that governs it. Without it: nobody. Each refusal is decided +//! before anything connects to the data table, so the fixture's database never has to exist. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn grant_select_on_public() -> Value { + json!({ + "target": {"kind": "schema", "schema": "public"}, + "change": {"type": "grant", "role": "analytics", "privileges": ["SELECT"], + "scope": "all_tables"}, + "statements": [r#"GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO "analytics""#] + }) +} + +async fn post_acl( + port: u16, + w_id: &str, + action: &str, + token: &str, +) -> anyhow::Result { + Ok(reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/datatable_acl/main/{action}" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&grant_select_on_public()) + .send() + .await?) +} + +/// A fork reaches the data table through a pointer: it may use it, never change what each role may +/// touch on it — not even as an admin of the fork. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_cannot_change_access_on_the_data_table_it_points_at( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "wm-fork-dt", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_member_who_is_not_an_admin_cannot_change_access( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "test-workspace", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +/// Not even reading, and not even on a data table that is not under roles — which any member +/// reaches, so only the edition stands between them and the instance's credentials. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn only_the_enterprise_edition_has_the_access_editor( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings + SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let read = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/datatable_acl/main?kind=database" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await?; + let mut responses = vec![("read", read)]; + for action in ["plan", "apply"] { + responses.push(( + action, + post_acl(port, "test-workspace", action, "SECRET_TOKEN").await?, + )); + } + for (action, resp) in responses { + assert_eq!(resp.status(), 400, "{action}"); + let body = resp.text().await?; + assert!( + body.contains("Data table roles are a Windmill Enterprise Edition feature"), + "{action}: {body}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs new file mode 100644 index 0000000000..4948a0098a --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -0,0 +1,1758 @@ +//! Who may connect to a data table as which role, across the two shapes an entry can take: one +//! that owns its database, and a fork's pointer at it. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +/// The `analytics` role's tenant list as stored, so a cascade can be observed directly. +async fn tenants(db: &Pool, w_id: &str) -> Vec { + let value: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main'->'permissions'->'roles'->'role1'->'tenants' + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_one(db) + .await + .unwrap(); + serde_json::from_value(value.unwrap_or(json!([]))).unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn freeing_a_principal_takes_its_datatable_tenant(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + assert_eq!( + tenants(&db, "test-workspace").await, + vec!["u/test-user-2", "g/analysts", "f/finance"] + ); + + let resp = authed( + client().delete(format!("{base}/groups/delete/analysts")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete group: {}", resp.text().await?); + + let resp = authed( + client().delete(format!("{base}/folders/delete/finance")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete folder: {}", resp.text().await?); + + let resp = authed( + client().delete(format!("{base}/users/delete/test-user-2")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete user: {}", resp.text().await?); + + // Leaving is the other way a membership ends, and there are two `/leave` routes — the one the + // UI and the generated client call is this one. A tenant left behind here comes back with the + // person on rejoin, or attaches to whoever takes the username next. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,main,permissions,roles,role1,tenants}', '["u/test-user-3"]'::jsonb) + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + let resp = authed( + client().post(format!("{base}/workspaces/leave")), + "SECRET_TOKEN_3", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "leave: {}", resp.text().await?); + // Nothing left naming a principal that no longer exists: a later group or account reusing one + // of those names must not inherit the access this one had. + assert!( + tenants(&db, "test-workspace").await.is_empty(), + "leaving kept the tenant: {:?}", + tenants(&db, "test-workspace").await + ); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_uses_the_data_table_it_points_at_but_never_administers_it( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let fork = format!("http://localhost:{port}/api/w/wm-fork-dt/workspaces"); + + // `test-user-2` is an admin of the fork and a plain member of the parent. The roles they can + // use are the ones the parent's tenants give them there, not what their fork admin bit says. + let resp = authed( + client().get(format!("{fork}/datatable_usable_roles/main")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["roles"], json!(["analytics"]), "{body}"); + assert_eq!(body["default_role"], "analytics"); + + // The drawer names the workspace that decides, and refuses to let the fork edit it. + let resp = authed( + client().get(format!("{fork}/datatable_permissions/main")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["governing_workspace_id"], "test-workspace"); + assert_eq!(body["editable"], false, "{body}"); + + let resp = authed( + client().post(format!("{fork}/datatable_permissions/main")), + "SECRET_TOKEN_2", + ) + .json(&json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]})) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a fork admin widened the parent's access" + ); + + // Nor by saving the settings form: the pointer is server-owned, so a payload naming the + // parent's database leaves the entry exactly as it was. + let resp = authed( + client().post(format!("{fork}/edit_datatable_config")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "settings": {"datatables": {"main": { + "database": {"resource_type": "instance", "resource_path": "dt_main"} + }}}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main' FROM workspace_settings WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + let entry = entry.unwrap(); + assert_eq!( + entry["reference"]["workspace_id"], "test-workspace", + "{entry}" + ); + assert!(entry["database"].is_null(), "{entry}"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_second_entry_on_the_same_database_is_reported_rather_than_governed( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A copy of the parent's entry, as a fork created before data table roles would hold. It keeps + // its own access, so the owner is told about it instead of being told it is covered. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": {"copy": { + "database": {"resource_type": "instance", "resource_path": "dt_main"}}}}'::jsonb + WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/datatable_permissions/main" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!( + body["ungoverned_reachers"], + json!([{"workspace_id": "wm-fork-dt", "datatable": "copy"}]), + "{body}" + ); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_resource_backed_data_table_cannot_be_put_under_roles( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A role is a login on Windmill's own cluster. A resource-backed data table dials a host the + // workspace admin chose, so accepting one here would hand that host a real cluster credential. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": {"byo": { + "database": {"resource_type": "postgresql", "resource_path": "u/test-user/pg"}}}}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let resp = authed( + client().get(format!("{base}/datatable_permissions/byo")), + "SECRET_TOKEN", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["supported"], false, "{body}"); + + let resp = authed( + client().post(format!("{base}/datatable_permissions/byo")), + "SECRET_TOKEN", + ) + .json(&json!({"permissioned": true, "default_role": "role1", + "roles": [{"id": "role1", "tenants": ["*"]}]})) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_renaming_its_own_entry_leaves_the_governing_bookkeeping_alone( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // The parent's migration definitions. A rename or delete through the fork's settings form + // resolves through the pointer, so without a guard it would relabel or wipe these. + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) + VALUES ('test-workspace', 'main', 1, 'init', 'SELECT 1')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "settings": {"datatables": {}}, + "renames": [], + "deleted_datatables": ["main"] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let left: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM datatable_migrations WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(left, 1, "the fork's delete reached the parent's migrations"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_caller_who_is_not_a_member_of_the_governing_workspace_reaches_nothing( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A fork member who was never added to the parent. Their fork membership says nothing there, + // and the email lookup that would evaluate them as a member of it finds no row. + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, role) + VALUES ('wm-fork-dt', 'test3@windmill.dev', 'test-user-3', false, 'User')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/datatable_usable_roles/main" + )), + "SECRET_TOKEN_3", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["roles"], json!([]), "{body}"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_caller_with_no_identity_reaches_a_permissioned_data_table_not_at_all( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + + initialize_tracing().await; + // The compatibility story for an agent worker that predates data table roles and sends no job + // id: it keeps resolving an unpermissioned data table, and is refused on a permissioned one + // rather than handed an unattributed admin connection. + let refused = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::NoIdentity, + ) + .await; + assert!(refused.is_err(), "an unidentified caller was let in"); + + sqlx::query( + "UPDATE workspace_settings + SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Postgres roles are cluster-wide and this cluster is shared with every other test database, + // so the names have to be unique to this run. + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let names = [format!("wmtest_a_{suffix}"), format!("wmtest_b_{suffix}")]; + + // The cluster DDL is not visible to another transaction until commit, so without the lock both + // of these pass their `pg_roles` existence check and one loses — leaving a live cluster login + // the catalog never recorded. + let create = |name: String| async move { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + Ok::<_, anyhow::Error>((status, body)) + }; + let outcome = async { + let (a, b) = tokio::join!(create(names[0].clone()), create(names[1].clone())); + let (a, b) = (a?, b?); + assert_eq!(a.0, 200, "{}", a.1); + assert_eq!(b.0, 200, "{}", b.1); + + let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let recorded: Vec<&str> = catalog.values().map(|r| r.name.as_str()).collect(); + for name in &names { + assert!( + recorded.contains(&name.as_str()), + "{name} is a live cluster login the catalog forgot: {recorded:?}" + ); + } + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide, so they outlive this test's throwaway database. Dropped whatever + // happened above — a failing run is exactly the one that created them and did not record them. + for name in &names { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let name = format!("wmtest_del_{suffix}"); + + let outcome = async { + let created: Value = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name })) + .send() + .await? + .error_for_status()? + .json() + .await?; + let id = created["id"].as_str().unwrap().to_string(); + + // Each database's pass commits on its own, so one that cannot be reached fails the delete + // after the others may already have stripped the role. + sqlx::query( + "UPDATE global_settings SET value = jsonb_set(value, '{databases,wm_unreachable}', '{}') + WHERE name = 'custom_instance_pg_databases'", + ) + .execute(&db) + .await?; + + let resp = authed( + client().delete(format!( + "http://localhost:{port}/api/settings/datatable_roles/{id}" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + + let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let role = catalog + .get(&id) + .expect("a failed delete keeps the entry to retry"); + assert!( + !role.enabled, + "a half-deleted role is still enabled in the catalog" + ); + let can_login: bool = + sqlx::query_scalar("SELECT rolcanlogin FROM pg_roles WHERE rolname = $1") + .bind(&name) + .fetch_one(&db) + .await?; + assert!(!can_login, "a half-deleted role can still log in"); + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + outcome +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn renaming_a_governing_data_table_carries_its_forks( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // A pointer names the governing data table by name, so a rename that does not follow leaves + // every fork resolving to nothing — the data table vanishes from their pickers and their jobs + // stop, with nothing in the renaming workspace to suggest why. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ + "settings": {"datatables": {"renamed": { + "database": {"resource_type": "instance", "resource_path": "dt_main"} + }}}, + "renames": [{"from": "main", "to": "renamed"}], + "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main' FROM workspace_settings WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + let entry = entry.unwrap(); + assert_eq!(entry["reference"]["datatable"], "renamed", "{entry}"); + + // And it still resolves, which is the thing the fork actually cares about. + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/datatable_usable_roles/main" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_rename_has_to_match_the_save_it_claims_to_describe( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = + format!("http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config"); + + let instance = + |path: &str| json!({"database": {"resource_type": "instance", "resource_path": path}}); + + // Fork pointers are rewritten from the rename list, so a rename nobody performed moves every + // fork of one data table onto another. `main` survives this save, so it was not renamed. + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ + "settings": {"datatables": {"main": instance("dt_main"), "decoy": instance("dt_two")}}, + "renames": [{"from": "main", "to": "decoy"}], + "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "a forged rename was accepted"); + + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main'->'reference' FROM workspace_settings + WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + assert_eq!( + entry.unwrap()["datatable"], + "main", + "the fork was repointed anyway" + ); + + // A swap is two renames whose sources and targets cross. It cannot be done one at a time — + // `datatables` is keyed by name — so refusing it would be a regression, and applying the two + // in order without a temporary name would carry `main`'s pointers back to `main`. + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ + "settings": {"datatables": {"main": instance("dt_two"), "other": instance("dt_main")}}, + "renames": [{"from": "main", "to": "other"}, {"from": "other", "to": "main"}], + "deleted_datatables": [] + })) + .send() + .await?; + // `other` does not exist yet, so this particular pair is still refused — the swap shape is + // covered by the pair below, which starts from two real data tables. + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_two"}}'::jsonb) + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ + "settings": {"datatables": {"main": instance("dt_two"), "other": instance("dt_main")}}, + "renames": [{"from": "main", "to": "other"}, {"from": "other", "to": "main"}], + "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a swap was refused: {}", + resp.text().await? + ); + + // The fork named `main`, which is now called `other`. + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main'->'reference' FROM workspace_settings + WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + assert_eq!( + entry.unwrap()["datatable"], + "other", + "the swap did not carry the pointer" + ); + Ok(()) +} + +async fn database_exists(db: &Pool, name: &str) -> anyhow::Result { + Ok( + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1)") + .bind(name) + .fetch_one(db) + .await?, + ) +} + +async fn workspace_exists(db: &Pool, id: &str) -> anyhow::Result { + Ok( + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM workspace WHERE id = $1)") + .bind(id) + .fetch_one(db) + .await?, + ) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_data_table_under_roles_is_cloned_only_with_its_grants( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + // The database a fork's copy of `main` goes into is named after the fork. Every request below + // is refused before it is created, so the cluster-wide name never collides with a sibling run. + let target = "wm_fork_copy__main"; + let fork = |token: &str, forked: Value| { + authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_fork" + )), + token, + ) + .json(&json!({"id": "wm-fork-copy", "name": "copy", "forked_datatables": [forked]})) + }; + + // Rows are a workspace admin's to copy, as they were before roles. + let resp = fork( + "SECRET_TOKEN_2", + json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_and_data"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + assert!(!database_exists(&db, target).await?); + + // Even the schema alone lists every table, which a member covered by no role cannot read in + // the parent. + let resp = fork( + "SECRET_TOKEN_3", + json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_only"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + assert!(!database_exists(&db, target).await?); + + // Refused in the first phase too, before a git branch is created for a fork that cannot be. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_workspace_fork_branch" + )), + "SECRET_TOKEN_3", + ) + .json( + &json!({"id": "wm-fork-copy", "name": "copy", "forked_datatables": [ + {"name": "main", "new_dbname": target, "fork_behavior": "schema_only"} + ]}), + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + + // A database the request did not create — whatever the data table, under roles or not — may be + // a copy left behind by a deleted fork, which the entry would reach without its governance. + let resp = fork( + "SECRET_TOKEN", + json!({"name": "other", "new_dbname": "wm_fork_copy__other"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("fork_behavior"), + "a database the fork did not create was taken" + ); + assert!(!workspace_exists(&db, "wm-fork-copy").await?); + + // The copy an admin asks for goes ahead in an edition that replays grants, and is refused + // before any database exists in one that does not. The fixture's database does not exist, so + // the dump fails either way — and leaves neither a database nor a fork behind. + let resp = fork( + "SECRET_TOKEN", + json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_only"}), + ) + .send() + .await?; + assert_ne!(resp.status(), 200); + let body = resp.text().await?; + #[cfg(all(feature = "private", feature = "enterprise"))] + assert!(body.contains("pg_dump"), "{body}"); + #[cfg(not(all(feature = "private", feature = "enterprise")))] + assert!(body.contains("Enterprise Edition"), "{body}"); + assert!(!database_exists(&db, target).await?); + assert!(!workspace_exists(&db, "wm-fork-copy").await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_clone_takes_its_roles_from_the_data_table_it_was_cloned_from( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,copy}', '{ + "database": {"resource_type": "instance", "resource_path": "wm_fork_dt__copy"}, + "governed_by": {"workspace_id": "test-workspace", "datatable": "main"}, + "forked_from": {} + }'::jsonb) WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let fork = format!("http://localhost:{port}/api/w/wm-fork-dt/workspaces"); + + // `test-user-2` administers the fork, and is only a tenant of `analytics` in the parent. + let resp = authed( + client().get(format!("{fork}/datatable_usable_roles/copy")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["roles"], json!(["analytics"]), "{body}"); + + let resp = authed( + client().get(format!("{fork}/datatable_permissions/copy")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["editable"], false, "{body}"); + assert_eq!(body["clone_of"]["workspace_id"], "test-workspace", "{body}"); + + // Nobody changes a clone's roles, a superadmin included: they are the source's. + for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] { + let resp = authed( + client().post(format!("{fork}/datatable_permissions/copy")), + token, + ) + .json(&json!({"permissioned": false})) + .send() + .await?; + assert_eq!(resp.status(), 400, "{token} changed a clone's roles"); + } + + // A settings save cannot clear the link. + let resp = authed( + client().post(format!("{fork}/edit_datatable_config")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "settings": {"datatables": {"main": {}, "copy": { + "database": {"resource_type": "instance", "resource_path": "wm_fork_dt__copy"} + }}}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let governed_by: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'copy'->'governed_by' FROM workspace_settings + WHERE workspace_id = 'wm-fork-dt'", + ) + .fetch_one(&db) + .await?; + assert_eq!(governed_by.unwrap()["datatable"], "main"); + + // Nor move it, a superadmin included: its grants were replayed into that database alone. + let resp = authed( + client().post(format!("{fork}/edit_datatable_config")), + "SECRET_TOKEN", + ) + .json(&json!({ + "settings": {"datatables": {"main": {}, "copy": { + "database": {"resource_type": "instance", "resource_path": "wm_fork_dt__elsewhere"} + }}}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // Nor reach its database through a second entry without roles, which would connect as `admin`. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ + "settings": {"datatables": { + "main": {"database": {"resource_type": "instance", "resource_path": "dt_main"}}, + "alias": {"database": {"resource_type": "instance", "resource_path": "wm_fork_dt__copy"}} + }}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + assert!(body.contains("without carrying those roles"), "{body}"); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_data_table_under_roles_is_not_copied_without_its_grants( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_pg_database" + )), + "SECRET_TOKEN", + ) + .json(&json!({"source": "datatable://main", "target_dbname": "wm_fork_dt_copy"})) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("under roles"), + "the fork's database was created for a copy that cannot happen" + ); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/import_pg_database" + )), + "SECRET_TOKEN", + ) + .json( + &json!({"source": "datatable://main", "target": "datatable://main", + "fork_behavior": "schema_only"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("under roles"), + "the copy was refused for some other reason" + ); + Ok(()) +} + +/// The fork's `forked_from` for one of its entries; `None` whether it is absent or `null`. +async fn forked_from_of(db: &Pool, name: &str) -> Option { + sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$1::text->'forked_from' + FROM workspace_settings WHERE workspace_id = 'wm-fork-dt'", + ) + .bind(name) + .fetch_one(db) + .await + .unwrap() + .filter(|v| !v.is_null()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_clone_stamp_is_carried_but_its_schema_baseline_advances( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Whether an entry is a clone is what marks its database droppable, so a save can neither + // stamp nor unstamp one. The schema baseline inside the stamp is what the fork's schema diff + // advances after applying a change; dropping it would offer that same change again. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": { + "clone": {"database": {"resource_type": "instance", "resource_path": "wm_fork_dt__clone"}, + "forked_from": {"schema": {}}}, + "plain": {"database": {"resource_type": "instance", "resource_path": "dt_plain"}}}}'::jsonb + WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let clone_db = json!({"resource_type": "instance", "resource_path": "wm_fork_dt__clone"}); + let plain_db = json!({"resource_type": "instance", "resource_path": "dt_plain"}); + let baseline = json!({"schema": {"public": {"orders": {"id": "int4"}}}}); + + let resp = authed(client().post(&url), "SECRET_TOKEN_2") + .json(&json!({"settings": {"datatables": { + "clone": {"database": clone_db, "forked_from": baseline}, + "plain": {"database": plain_db, "forked_from": {"schema": {}}} + }}})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!( + forked_from_of(&db, "clone").await, + Some(baseline.clone()), + "the schema diff's baseline did not advance" + ); + assert_eq!( + forked_from_of(&db, "plain").await, + None, + "a save stamped a clone" + ); + + let resp = authed(client().post(&url), "SECRET_TOKEN_2") + .json(&json!({"settings": {"datatables": { + "clone": {"database": clone_db}, "plain": {"database": plain_db} + }}})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!( + forked_from_of(&db, "clone").await, + Some(baseline), + "a save unstamped a clone" + ); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A replication stream reads every row whatever the roles grant, so a data table carries one + // or the other. An enabled trigger on it — here a fork's, through its pointer — keeps roles + // from being turned on, and disabling it is what lets them on. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + sqlx::query( + r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, mode) + VALUES ('u/test-user-2/fork_stream', 'u/test-user-2/s', false, 'wm-fork-dt', + 'test-user-2', 'datatable://main', 'slot_fork', 'pub_fork', 'u/test-user-2', + 'enabled')"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/datatable_permissions/main", + server.addr.port() + ); + let turn_on = json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]}); + + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text() + .await? + .contains("wm-fork-dt/u/test-user-2/fork_stream"), + "the refusal does not name the trigger to disable" + ); + + // Disabled, but its listener pinged just now and stops only at its next heartbeat. + sqlx::query( + "UPDATE postgres_trigger SET mode = 'disabled', server_id = NULL, last_server_ping = now() + WHERE path = 'u/test-user-2/fork_stream'", + ) + .execute(&db) + .await?; + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "roles went on while a disabled trigger's listener was still attached: {}", + resp.text().await? + ); + + sqlx::query( + "UPDATE postgres_trigger SET last_server_ping = now() - interval '20 seconds' + WHERE path = 'u/test-user-2/fork_stream'", + ) + .execute(&db) + .await?; + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn roles_going_on_wait_for_a_trigger_being_enabled(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + + // A trigger enable in flight: it holds the stream lock and its row is not committed yet, so a + // roles save that looked for streams now would miss it and its listener would connect to a + // data table it is about to be refused. + let mut enabling = db.begin().await?; + windmill_common::datatable_roles::lock_datatable_streams(&mut *enabling, false).await?; + sqlx::query( + r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, mode) + VALUES ('u/test-user-2/racing_stream', 'u/test-user-2/s', false, 'wm-fork-dt', + 'test-user-2', 'datatable://main', 'slot_race', 'pub_race', 'u/test-user-2', + 'enabled')"#, + ) + .execute(&mut *enabling) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/datatable_permissions/main", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]})) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "roles went on while a trigger was being enabled" + ); + enabling.commit().await?; + + let resp = save.await??; + assert_eq!(resp.status(), 400); + assert!( + resp.text() + .await? + .contains("wm-fork-dt/u/test-user-2/racing_stream"), + "the roles save missed the trigger enabled while it waited" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_stored_name_containing_a_question_mark_resolves_as_itself( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Names could contain `?` before they were restricted, and such an entry is still stored. + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,legacy?dt}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + + let resolve = |reference: &'static str| { + let db = db.clone(); + async move { + windmill_common::workspaces::parse_datatable_ref_for(&db, "test-workspace", reference) + .await + } + }; + assert_eq!(resolve("legacy?dt").await?, ("legacy?dt".to_string(), None)); + assert_eq!( + resolve("main?role=analytics").await?, + ("main".to_string(), Some("analytics".to_string())) + ); + assert!( + resolve("main?dt").await.is_err(), + "an unknown parameter was ignored" + ); + + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main?role=analytics}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + assert!( + resolve("main?role=analytics").await.is_err(), + "a reference naming both a stored data table and a role on another resolved to one of them" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_settings_save_dropping_a_governing_entry_names_the_forks_it_strands( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + // The whole map and no `deleted_datatables`, as a settings sync sends it. + let resp = authed( + client().post(format!( + "http://localhost:{}/api/w/test-workspace/workspaces/edit_datatable_config", + server.addr.port() + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": {} } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "{body}"); + let result: Value = serde_json::from_str(&body)?; + assert!( + result["stranded_references"] + .as_array() + .is_some_and(|refs| refs.iter().any(|r| r["workspace_id"] == "wm-fork-dt")), + "the fork left pointing at nothing was not named: {body}" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Whole-map saves with no `renames`, as a settings sync sends them. + let dt_main = + json!({ "database": { "resource_type": "instance", "resource_path": "dt_main" } }); + let dt_other = + json!({ "database": { "resource_type": "instance", "resource_path": "dt_other" } }); + for (case, w_id, datatables) in [ + ( + "a rename to a new name", + "test-workspace", + json!({ "main_renamed": dt_main, "other": dt_other }), + ), + ( + "an existing name repointed", + "test-workspace", + json!({ "other": dt_main }), + ), + ( + "another workspace's entry", + "wm-fork-dt", + json!({ "direct": dt_main }), + ), + ] { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": datatables } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "{case} reached the database under roles without them ({status}): {body}" + ); + } + + let still_governed: bool = sqlx::query_scalar( + "SELECT (datatable->'datatables'->'main') ? 'permissions' FROM workspace_settings + WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert!(still_governed, "the refused save still took effect"); + Ok(()) +} + +/// Browsing names the role it connects as, and a role the caller may not use is refused rather +/// than quietly listed as the default. The refusal is decided before connecting, so the fixture's +/// database never has to exist. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn browsing_as_a_role_the_caller_may_not_use_is_refused( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + // `test-user-2` is a tenant of `analytics` only. + let resp = authed( + client().get(format!( + "{base}/list_datatable_tables?role_for=main&role=admin" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + let entry = body + .as_array() + .and_then(|a| a.iter().find(|e| e["datatable_name"] == "main")) + .expect("main is listed"); + assert_eq!(entry["usable_roles"], json!(["analytics"]), "{entry}"); + assert_eq!(entry["default_role"], "analytics", "{entry}"); + assert_eq!(entry["permissioned"], true, "{entry}"); + assert_eq!(entry["instance"], true, "{entry}"); + let error = entry["error"].as_str().unwrap_or_default(); + assert!( + error.contains("Not allowed to use role 'admin'"), + "listed as another role than the one asked for: {entry}" + ); + + let resp = authed( + client().get(format!( + "{base}/get_datatable_table_schema?datatable_name=main&schema_name=public&table_name=t&role=admin" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + text.contains("Not allowed to use role 'admin'"), + "{status}: {text}" + ); + + // A role means nothing without the data table it belongs to. + let resp = authed( + client().get(format!("{base}/list_datatable_tables?role=analytics")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_alias_saved_elsewhere_waits_for_roles_going_on_for_its_database( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + // Roles going on for `dt_other`, not committed yet: it holds only its own workspace's settings + // row, so an alias saved from another workspace that looked for roles now would miss them. + let enabling = { + let mut tx = db.begin().await?; + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + ["dt_other"], + ) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,other,permissions}', + '{"default_role": "admin", "roles": {"admin": {"tenants": ["*"]}}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&mut *tx) + .await?; + tx + }; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ "settings": { "datatables": { + "direct": { "database": { "resource_type": "instance", "resource_path": "dt_other" } } + } } })) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "an alias was saved while roles were going on for its database" + ); + enabling.commit().await?; + + let resp = save.await??; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "the alias reached the database whose roles went on while it waited ({status}): {body}" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_waits_for_a_rename_of_the_data_table_it_keeps( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A rename in flight holds the parent's settings row and moves only the pointers it can see; a + // fork being created is invisible to it, so the fork has to read the name the rename commits. + let mut renaming = db.begin().await?; + sqlx::query( + "SELECT 1 FROM workspace_settings WHERE workspace_id = 'test-workspace' FOR UPDATE", + ) + .execute(&mut *renaming) + .await?; + sqlx::query( + "UPDATE workspace_settings SET datatable = jsonb_set(datatable #- '{datatables,main}', + '{datatables,renamed}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&mut *renaming) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/create_fork", + server.addr.port() + ); + let fork = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ "id": "wm-fork-race", "name": "race", "color": "#0000ff" })) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !fork.is_finished(), + "the fork copied the parent's data tables while a rename held them" + ); + renaming.commit().await?; + + let resp = fork.await??; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let datatables: Option = sqlx::query_scalar( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = 'wm-fork-race'", + ) + .fetch_one(&db) + .await?; + let datatables = datatables.unwrap(); + assert_eq!( + datatables["renamed"]["reference"], + json!({ "workspace_id": "test-workspace", "datatable": "renamed" }), + "{datatables}" + ); + Ok(()) +} + +async fn enable_migrations_on_main(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main,migrations_enabled}', 'true') + WHERE workspace_id = 'test-workspace'", + ) + .execute(db) + .await?; + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_migration_is_written_only_by_who_may_run_it(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + enable_migrations_on_main(&db).await?; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let create = |token: &str, body: Value| { + authed( + client() + .post(format!("{base}/create_datatable_migration/main")) + .json(&body), + token, + ) + .send() + }; + let upsert = |token: &str, code_up: &str| { + authed( + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&json!({"timestamp": 20260101000000i64, "name": "m", "code_up": code_up})), + token, + ) + .send() + }; + + // `test-user-2` holds `analytics` and nothing else, so of a migration they may store only + // what they could run: SQL that declares no role runs as admin, in the up or the down. + for body in [ + json!({"name": "m", "code_up": "DROP TABLE t"}), + json!({"name": "m", "code_up": "-- role analytics\nSELECT 1", "code_down": "DROP TABLE t"}), + ] { + let resp = create("SECRET_TOKEN_2", body.clone()).await?; + assert_eq!(resp.status(), 401, "{body}: {}", resp.text().await?); + } + let resp = create( + "SECRET_TOKEN_2", + json!({"name": "m", "code_up": "-- role analytics\nSELECT 1"}), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An admin's migration re-pushed as stored writes nothing, so it passes. Replacing it removes + // SQL they could not run, even with SQL they could, so it is refused like deleting it. + let resp = upsert("SECRET_TOKEN", "DROP TABLE t").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let resp = upsert("SECRET_TOKEN_2", "DROP TABLE t").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + for code_up in ["DROP TABLE u", "-- role analytics\nSELECT 1"] { + let resp = upsert("SECRET_TOKEN_2", code_up).await?; + assert_eq!(resp.status(), 401, "{code_up}: {}", resp.text().await?); + } + let resp = authed( + client().delete(format!( + "{base}/delete_datatable_migration/main/20260101000000" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + + let code_up: String = sqlx::query_scalar( + "SELECT code_up FROM datatable_migrations + WHERE workspace_id = 'test-workspace' AND timestamp = 20260101000000", + ) + .fetch_one(&db) + .await?; + assert_eq!(code_up, "DROP TABLE t"); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_operator_neither_writes_nor_runs_a_migration(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + enable_migrations_on_main(&db).await?; + // Off roles, where every role check passes everyone, so only the operator guard is left to + // refuse. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + // A token of its own: the auth cache is process-wide, and the other tests here authenticate + // `SECRET_TOKEN_3` in this workspace as a non-operator. + sqlx::raw_sql( + "UPDATE usr SET operator = true, role = 'Operator' + WHERE workspace_id = 'test-workspace' AND username = 'test-user-3'; + INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('DT_OPERATOR_TOKEN'::bytea), 'hex'), 'DT_OPERATO', + 'DT_OPERATOR_TOKEN', 'test3@windmill.dev', 'operator', false);", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let migration = json!({"timestamp": 20260101000000i64, "name": "m", "code_up": "SELECT 1"}); + + for request in [ + client() + .post(format!("{base}/create_datatable_migration/main")) + .json(&migration), + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&migration), + client().delete(format!( + "{base}/delete_datatable_migration/main/20260101000000" + )), + client().post(format!("{base}/run_datatable_migrations/main")), + client().post(format!("{base}/rollback_datatable_migrations/main")), + client().post(format!("{base}/generate_initial_datatable_migration/main")), + ] { + let resp = authed(request, "DT_OPERATOR_TOKEN").send().await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + status == 401 && text.contains("Operators cannot"), + "{status}: {text}" + ); + } + + // The same write from a developer is accepted: what refused the operator is theirs alone. + let resp = authed( + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&migration), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn every_roles_route_is_an_enterprise_feature(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let api = format!("http://localhost:{}/api", server.addr.port()); + let dt = format!("{api}/w/test-workspace/workspaces"); + + for (method, url, body) in [ + ( + reqwest::Method::GET, + format!("{api}/settings/datatable_roles"), + None, + ), + ( + reqwest::Method::POST, + format!("{api}/settings/datatable_roles"), + Some(json!({ "name": "wmtest_ce" })), + ), + ( + reqwest::Method::POST, + format!("{api}/settings/datatable_roles/role1"), + Some(json!({ "enabled": false })), + ), + ( + reqwest::Method::DELETE, + format!("{api}/settings/datatable_roles/role1"), + None, + ), + ( + reqwest::Method::GET, + format!("{dt}/datatable_permissions/main"), + None, + ), + ( + reqwest::Method::POST, + format!("{dt}/datatable_permissions/main"), + Some(json!({ "permissioned": false })), + ), + ( + reqwest::Method::GET, + format!("{dt}/datatable_usable_roles/main"), + None, + ), + ] { + let mut request = authed(client().request(method.clone(), &url), "SECRET_TOKEN"); + if let Some(body) = body { + request = request.json(&body); + } + let resp = request.send().await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + status == 400 && text.contains(ENTERPRISE_REFUSAL), + "{method} {url} answered {status}: {text}" + ); + } + + // Refused, not acted on: the catalog row and the data table's roles are where they were. + let untouched: (i64, bool) = sqlx::query_as( + "SELECT (SELECT count(*) FROM datatable_role), + (datatable->'datatables'->'main') ? 'permissions' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(untouched, (1, true)); + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn without_the_enterprise_edition_a_data_table_under_roles_is_refused_a_connection( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + initialize_tracing().await; + + // Saved under roles, as an enterprise build left it: refused whoever asks, never `admin`. + for access in [DatatableAccess::Unchecked, DatatableAccess::NoIdentity] { + let err = get_datatable_resource_from_db(&db, "test-workspace", "main", None, access) + .await + .expect_err("a data table under roles resolved"); + assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); + } + + // Not under roles, it resolves as it always has, including when `admin` is named — which every + // migration does; naming any other role on it is refused. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + for role in [None, Some("admin")] { + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + role, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + } + let err = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + Some("analytics"), + DatatableAccess::Unchecked, + ) + .await + .expect_err("a named role resolved"); + assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); + Ok(()) +} + +/// A save blocked on an instance database's lock is not a user of it until it commits one: the +/// cleanup for a database whose setup failed lets that save through and reads what it left. +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn cleanup_waits_out_a_save_racing_it_for_an_instance_database( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let key = "instance_database:dt_probe"; + + // Queues cleanup behind `holder`, then a save behind cleanup, so cleanup takes the lock with + // the save already waiting on it — the ordering the waiter check is for. + let race = |commit: bool| { + let db = db.clone(); + async move { + let mut holder = db.acquire().await?; + sqlx::query("SELECT pg_advisory_lock(hashtext($1))") + .bind(key) + .execute(&mut *holder) + .await?; + + let cleanup = tokio::spawn({ + let db = db.clone(); + async move { windmill_common::drop_unused_instance_database(&db, "dt_probe").await } + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + let save = tokio::spawn({ + let db = db.clone(); + async move { + let mut tx = db.begin().await?; + windmill_common::lock_instance_databases(&mut tx, ["dt_probe"]).await?; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,probe}', + '{"database": {"resource_type": "instance", "resource_path": "dt_probe"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&mut *tx) + .await?; + if commit { + tx.commit().await?; + } else { + tx.rollback().await?; + } + Ok::<_, anyhow::Error>(()) + } + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + sqlx::query("SELECT pg_advisory_unlock(hashtext($1))") + .bind(key) + .execute(&mut *holder) + .await?; + save.await??; + Ok::<_, anyhow::Error>(cleanup.await??) + } + }; + + assert!( + matches!(race(false).await?, windmill_common::Cleanup::Dropped), + "a save that rolled back kept the database, whose name then blocks every retry" + ); + let kept = race(true).await?; + let windmill_common::Cleanup::InUse(users) = kept else { + anyhow::bail!("the database was dropped under a save that committed a reference to it") + }; + assert_eq!(users, vec!["test-workspace".to_string()]); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql b/backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql new file mode 100644 index 0000000000..55d50d212c --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql @@ -0,0 +1,45 @@ +-- A data table under roles in `test-workspace`, and a fork whose entry points at it rather than +-- carrying a copy. `test-user-2` is a non-admin of the parent and an admin of the fork: the shape +-- the pointer exists for. + +-- Empty registry: role provisioning grants CONNECT on every database named here, and the data +-- table's `dt_main` is a name in workspace settings, not a database that exists. +INSERT INTO global_settings (name, value) VALUES + ('custom_instance_pg_databases', '{"user_pwd": "pw", "databases": {}}'::jsonb) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; + +INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ('role1', 'analytics', true, 'pw'); + +UPDATE workspace_settings SET datatable = '{ + "datatables": { + "main": { + "database": {"resource_type": "instance", "resource_path": "dt_main"}, + "permissions": { + "default_role": "role1", + "roles": { + "admin": {"tenants": []}, + "role1": {"tenants": ["u/test-user-2", "g/analysts", "f/finance"]} + } + } + } + } +}'::jsonb WHERE workspace_id = 'test-workspace'; + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace', 'analysts', 'Analysts', '{}'); +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES + ('test-workspace', 'finance', 'finance', '{}', '{}'); + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('wm-fork-dt', 'fork of test-workspace', 'test2@windmill.dev', 'test-workspace'); +INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('wm-fork-dt', 'cloud', 'test-key'); +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('wm-fork-dt', 'all', 'All users', '{}'); +INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES + ('wm-fork-dt', 'test2@windmill.dev', 'test-user-2', true, 'Admin'); + +INSERT INTO workspace_settings (workspace_id, datatable) VALUES ('wm-fork-dt', '{ + "datatables": { + "main": {"reference": {"workspace_id": "test-workspace", "datatable": "main"}} + } +}'::jsonb); diff --git a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs index dae02f3e5a..bc6c10f1ee 100644 --- a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs +++ b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs @@ -311,3 +311,130 @@ async fn test_fork_keeps_only_resolvable_on_behalf_of(db: Pool) -> any Ok(()) } + +/// Apps, schedules, triggers and their drafts cannot drop an identity the way scripts and flows +/// do, so one naming nobody in the fork goes to its creator while one that still resolves stays. +/// Forked as an admin, whose app policies the clone otherwise keeps. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_fork_repoints_unresolvable_identities(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base_url = format!("http://localhost:{}/api", server.addr.port()); + + let stranger = json!({ + "on_behalf_of": "u/test-user-2", + "on_behalf_of_email": "test2@windmill.dev", + "execution_mode": "publisher", + }); + sqlx::query( + "INSERT INTO app (workspace_id, path, summary, policy, versions) + VALUES ('test-workspace', 'u/test-user/stranger', '', $1, '{}'), + ('test-workspace', 'u/test-user/group', '', $2, '{}')", + ) + .bind(&stranger) + .bind(json!({ + "on_behalf_of": "g/all", + "on_behalf_of_email": "group-all@windmill.dev", + "execution_mode": "publisher", + })) + .execute(&db) + .await?; + // The clone re-aggregates `versions` from `app_version`, and the column is NOT NULL. + sqlx::query( + "WITH v AS ( + INSERT INTO app_version (app_id, value, created_by) + SELECT id, '{}'::json, 'test-user' FROM app WHERE workspace_id = 'test-workspace' + RETURNING id, app_id + ) + UPDATE app SET versions = ARRAY[v.id] FROM v WHERE app.id = v.app_id", + ) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO draft (workspace_id, path, typ, value, created_at, email) + VALUES ('test-workspace', 'u/test-user/stranger', 'raw_app', $1::json, NOW(), 'test@windmill.dev'), + ('test-workspace', 'u/test-user/stranger', 'trigger_websocket', $2::json, NOW(), 'test@windmill.dev'), + ('test-workspace', 'u/test-user/nul', 'raw_app', $3::json, NOW(), 'test@windmill.dev')", + ) + .bind(json!({ "policy": stranger })) + .bind(json!({ "permissioned_as": "u/test-user-2" })) + // Saved before drafts were stripped of NULs: any jsonb parse of it raises, so it must be + // skipped rather than abort the fork. Built from parts because a NUL escape can't sit in source. + .bind(format!( + r#"{{"policy":{{"on_behalf_of":"u/test-user-2"}},"files":{{"f":"a{}u0000"}}}}"#, + "\\" + )) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, schedule, script_path, email, permissioned_as, enabled) + VALUES ('test-workspace', 'u/test-user/stranger', 'test-user', '0 0 * * * *', 'u/test-user/s', 'test2@windmill.dev', 'u/test-user-2', false)", + ) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, edited_by, permissioned_as, mode) + VALUES ('test-workspace', 'u/test-user/stranger', 'ws://localhost', 'u/test-user/s', false, 'test-user', 'u/test-user-2', 'disabled')", + ) + .execute(&db) + .await?; + + let resp = reqwest::Client::new() + .post(format!( + "{base_url}/w/test-workspace/workspaces/create_fork" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "id": "wm-fork-repoint", "name": "Fork", "color": "#0000ff" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "creating the fork: {}", + resp.text().await? + ); + + let text = |sql: &'static str| sqlx::query_scalar::<_, String>(sql).fetch_one(&db); + assert_eq!( + text("SELECT (policy->>'on_behalf_of') || ' ' || (policy->>'on_behalf_of_email') FROM app WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/stranger'").await?, + "u/test-user test@windmill.dev" + ); + assert_eq!( + text("SELECT policy->>'on_behalf_of' FROM app WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/group'").await?, + "g/all" + ); + assert_eq!( + text("SELECT value->'policy'->>'on_behalf_of' FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/stranger' AND typ = 'raw_app'").await?, + "u/test-user" + ); + // `clone_drafts` strips a NUL escape as it copies, so the row reaches the fork + // parseable and the repoint below reaches it like any other draft's. The rule this + // guards is that the fork completes and no identity naming nobody survives it; the + // skip only ever existed because `to_jsonb` raises on a value still holding one. + assert_eq!( + text("SELECT CASE WHEN strpos(value::text, 'u/test-user-2') > 0 THEN 'kept' ELSE 'rewritten' END FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/nul'").await?, + "rewritten" + ); + // And it arrives without the poison that made it a special case. + assert_eq!( + text("SELECT CASE WHEN position(chr(92) || 'u0000' in value::text) > 0 THEN 'poisoned' ELSE 'clean' END FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/nul'").await?, + "clean" + ); + assert_eq!( + text("SELECT value->>'permissioned_as' FROM draft WHERE workspace_id = 'wm-fork-repoint' AND typ = 'trigger_websocket'").await?, + "u/test-user" + ); + assert_eq!( + text("SELECT permissioned_as || ' ' || email FROM schedule WHERE workspace_id = 'wm-fork-repoint'").await?, + "u/test-user test@windmill.dev" + ); + assert_eq!( + text( + "SELECT permissioned_as FROM websocket_trigger WHERE workspace_id = 'wm-fork-repoint'" + ) + .await?, + "u/test-user" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/fork_members.rs b/backend/windmill-api-integration-tests/tests/fork_members.rs new file mode 100644 index 0000000000..5f4baced1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fork_members.rs @@ -0,0 +1,75 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// With `add_admins_and_developers_to_forks` on, a fork starts with the parent's admins and +/// developers at their parent role, even when a developer forks it; operators are left out. The +/// copies are manual members: a parent membership that came from an instance group must not carry +/// that provenance into a fork that does not configure the group. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_fork_adds_parent_admins_and_developers(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base_url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let client = reqwest::Client::new(); + + sqlx::query( + "UPDATE usr SET operator = true WHERE workspace_id = 'test-workspace' AND username = 'test-user-3'", + ) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, added_via) + VALUES ('test-workspace', 'test4@windmill.dev', 'test-user-4', false, + '{\"source\": \"instance_group\", \"group\": \"devs\"}')", + ) + .execute(&db) + .await?; + + let resp = client + .post(format!( + "{base_url}/edit_add_admins_and_developers_to_forks" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "add_admins_and_developers_to_forks": true })) + .send() + .await?; + assert!( + resp.status().is_success(), + "enabling the setting: {}", + resp.text().await? + ); + + let resp = client + .post(format!("{base_url}/create_fork")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .json(&json!({ "id": "wm-fork-team", "name": "Team fork" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "creating the fork: {}", + resp.text().await? + ); + + let members: Vec<(String, bool, bool)> = sqlx::query_as( + "SELECT username, is_admin, added_via IS NULL FROM usr + WHERE workspace_id = 'wm-fork-team' ORDER BY username", + ) + .fetch_all(&db) + .await?; + assert_eq!( + members, + vec![ + ("test-user".to_string(), true, true), + ("test-user-2".to_string(), false, true), + ("test-user-4".to_string(), false, true), + ] + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index f86522aae3..26e5caeea1 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -913,3 +913,140 @@ async fn test_preserve_orphaned_members_migration(db: Pool) -> anyhow: Ok(()) } + +/// A membership row whose value is not an email (an IdP object id a SCIM sync stored before +/// member values were validated) must not break the workspace's instance-group save: the +/// reconciler skips it and still provisions the valid members. The admin endpoint refuses to +/// add such a value in the first place. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_member_that_is_not_an_email_is_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + const ENTRA_OBJECT_ID: &str = "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed"; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "entra_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create"); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": "kept@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser"); + + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": ENTRA_OBJECT_ID })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value that is not an email" + ); + let too_wide = format!("{}@example.com", "a".repeat(244)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": too_wide })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value wider than the email columns" + ); + // A valid address whose local part is wider than the username columns: the derived + // username is cut to fit rather than failing the promotion. + let long_local_part = format!("{}@example.com", "a".repeat(60)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": long_local_part })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser long local part"); + + sqlx::query("INSERT INTO email_to_igroup (email, igroup) VALUES ($1, 'entra_grp')") + .bind(ENTRA_OBJECT_ID) + .execute(&db) + .await?; + + // A member whose address only the wider `proper_email` of `usr` accepts, already + // provisioned through the group: reconciliation must keep and re-role them, since + // removal destroys their drafts, inputs and permissions. + sqlx::raw_sql( + r#" + INSERT INTO email_to_igroup (email, igroup) VALUES ('"quoted"@example.com', 'entra_grp'); + INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ('test-workspace', 'quoted', '"quoted"@example.com', false, true, + '{"source": "instance_group", "group": "entra_grp"}'::jsonb); + "#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["entra_grp"], + "roles": { "entra_grp": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let mut members: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + members.sort(); + assert_eq!( + members, + vec![ + ("\"quoted\"@example.com".to_string(), false), + (long_local_part.clone(), false), + ("kept@example.com".to_string(), false), + ], + "valid members provisioned and existing member kept, all as developers; non-email one skipped" + ); + + // A full import carrying the same rows: the object id is dropped, the address only + // `proper_email` accepts is kept, and neither member loses their workspace row. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([{ + "name": "entra_grp", + "emails": ["kept@example.com", "\"quoted\"@example.com", long_local_part, ENTRA_OBJECT_ID] + }])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + let mut stored: Vec = + sqlx::query_scalar("SELECT email FROM email_to_igroup WHERE igroup = 'entra_grp'") + .fetch_all(&db) + .await?; + stored.sort(); + assert_eq!( + stored, + vec![ + "\"quoted\"@example.com".to_string(), + long_local_part.clone(), + "kept@example.com".to_string(), + ], + "import drops the object id and keeps the rest" + ); + let mut after_import: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + after_import.sort(); + assert_eq!(after_import, members, "import must not evict either member"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/login_link.rs b/backend/windmill-api-integration-tests/tests/login_link.rs new file mode 100644 index 0000000000..5c3a231a48 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/login_link.rs @@ -0,0 +1,192 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn login_link_is_single_use_and_same_origin(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + let mint = |token: &'static str, body: serde_json::Value| { + client() + .post(format!("{base}/users/login_links")) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + }; + + // Only a superadmin mints. + let resp = mint("SECRET_TOKEN_2", json!({"email": "test2@windmill.dev"})).await?; + assert_eq!(resp.status(), 401); + + // A superadmin account is never a valid target: the minting credential must not + // become an instance-wide role. + let resp = mint("SECRET_TOKEN", json!({"email": "test@windmill.dev"})).await?; + assert_eq!(resp.status(), 400); + + // An off-origin destination is refused before anything is minted. + let resp = mint( + "SECRET_TOKEN", + json!({"email": "test2@windmill.dev", "rd": "https://evil.example/"}), + ) + .await?; + assert_eq!(resp.status(), 400); + + let resp = mint( + "SECRET_TOKEN", + json!({"email": "test2@windmill.dev", "rd": "/user/workspaces?x=1"}), + ) + .await?; + assert_eq!(resp.status(), 201); + let link = resp.json::().await?; + let path = link["url"] + .as_str() + .unwrap() + .split_once("/api") + .unwrap() + .1 + .to_string(); + let consume_url = format!("{base}{path}"); + + // A promotion inside the link's window is re-checked at open time: no session, + // and the link is not spent while the account is privileged. + sqlx::query("UPDATE password SET super_admin = true WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!( + resp.headers()["location"], + "/user/login_link_expired?reason=invalid" + ); + assert!(resp.headers().get("set-cookie").is_none()); + sqlx::query("UPDATE password SET super_admin = false WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + + // First open: session cookie for the target account, redirected to the stored rd. + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!(resp.headers()["location"], "/user/workspaces?x=1"); + assert_eq!(resp.headers()["referrer-policy"], "no-referrer"); + let cookie = resp + .headers() + .get_all("set-cookie") + .iter() + .map(|c| c.to_str().unwrap().to_string()) + .find(|c| c.starts_with("token=")) + .expect("session cookie"); + assert!(cookie.contains("HttpOnly")); + let session = cookie + .split(';') + .next() + .unwrap() + .trim_start_matches("token=") + .to_string(); + let resp = client() + .get(format!("{base}/users/whoami")) + .header("Authorization", format!("Bearer {session}")) + .send() + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?["email"], + "test2@windmill.dev" + ); + + // Second open: burned, no cookie, bounced to the explanation page. + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!( + resp.headers()["location"], + "/user/login_link_expired?reason=used" + ); + assert!(resp.headers().get("set-cookie").is_none()); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn login_link_mint_can_require_a_login_type(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + let mint = || { + client() + .post(format!("{base}/users/login_links")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({"email": "test2@windmill.dev", "require_login_type": "pending_oauth"})) + .send() + }; + + // A password account is not the account the caller created: no link. + let resp = mint().await?; + assert_eq!(resp.status(), 409); + assert!(resp.text().await?.contains("login_type_mismatch")); + + sqlx::query( + "UPDATE password SET login_type = 'pending_oauth', password_hash = NULL WHERE email = 'test2@windmill.dev'", + ) + .execute(&db) + .await?; + let resp = mint().await?; + assert_eq!(resp.status(), 201); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn cloud_trial_offer_go_refuses_a_job_token(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + + // The answer is a signed-in portal login for the account, so the job-token check + // must come before every other gate: a script holding `$WM_TOKEN` is refused outright, + // where a browser session reaches the next check (off cloud, "no offer"). + let job_id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user-2', 'u/test-user-2', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job_id) + .execute(&db) + .await?; + let job_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "job", + 600, + "test2@windmill.dev", + &job_id, + None, + None, + ) + .await?; + + let go = |token: String| { + client() + .post(format!("{base}/users/cloud_trial_offer/go")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let resp = go(job_token).await?; + assert_eq!(resp.status(), 403); + assert!(resp.text().await?.contains("job token")); + + let resp = go("SECRET_TOKEN_2".to_string()).await?; + assert_eq!(resp.status(), 404); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/max_token_expiration.rs b/backend/windmill-api-integration-tests/tests/max_token_expiration.rs new file mode 100644 index 0000000000..b748ce49a1 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/max_token_expiration.rs @@ -0,0 +1,239 @@ +//! `max_token_expiration_days`: the instance-wide ceiling on how far ahead a token a caller +//! picks the lifetime of may expire, and the service-account exemption. + +use serde_json::json; +use sqlx::types::chrono::{DateTime, Utc}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const DAY: u64 = 24 * 60 * 60; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn from_now(secs: u64) -> DateTime { + Utc::now() + std::time::Duration::from_secs(secs) +} + +async fn set_max(db: &Pool, value: serde_json::Value) { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('max_token_expiration_days', $1) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(value) + .execute(db) + .await + .unwrap(); +} + +/// Mints as `test2@windmill.dev`, a plain member of `test-workspace`. +async fn create_token(port: u16, body: serde_json::Value) -> reqwest::Response { + client() + .post(format!("http://localhost:{port}/api/users/tokens/create")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .json(&body) + .send() + .await + .unwrap() +} + +async fn stored_expiration(db: &Pool, label: &str) -> Option> { + sqlx::query_scalar::<_, Option>>("SELECT expiration FROM token WHERE label = $1") + .bind(label) + .fetch_one(db) + .await + .unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_max_token_expiration_days_shortens_user_tokens( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = create_token(port, json!({ "label": "unset" })).await; + assert_eq!(resp.status(), 201); + assert_eq!( + stored_expiration(&db, "unset").await, + None, + "with no setting a token may still have no expiration" + ); + + // Refused at write time: the token routes can only read a value they cannot parse as no + // ceiling at all. + let resp = client() + .post(format!( + "http://localhost:{port}/api/settings/global/max_token_expiration_days" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "value": 7.5 })) + .send() + .await?; + assert_eq!(resp.status(), 400); + + set_max(&db, json!(7)).await; + + // The token form reads the ceiling as whoever is creating the token, usually not a + // superadmin, so it can offer only expirations the server would keep. + let resp = client() + .get(format!( + "http://localhost:{port}/api/settings/global/max_token_expiration_days" + )) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .send() + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(resp.text().await?, "7"); + + let resp = create_token(port, json!({ "label": "none asked" })).await; + assert_eq!(resp.status(), 201); + let expiration = stored_expiration(&db, "none asked") + .await + .expect("a token asking for no expiration gets the ceiling"); + assert!( + expiration > from_now(6 * DAY) && expiration <= from_now(7 * DAY), + "expected the 7 day ceiling, got {expiration}" + ); + + let resp = create_token( + port, + json!({ "label": "past the ceiling", "expiration": from_now(30 * DAY) }), + ) + .await; + assert_eq!(resp.status(), 201); + let expiration = stored_expiration(&db, "past the ceiling").await.unwrap(); + assert!( + expiration > from_now(6 * DAY) && expiration <= from_now(7 * DAY), + "expected an expiration past the ceiling to be shortened to it, got {expiration}" + ); + + let resp = create_token( + port, + json!({ "label": "within", "expiration": from_now(3 * DAY) }), + ) + .await; + assert_eq!(resp.status(), 201); + let expiration = stored_expiration(&db, "within").await.unwrap(); + assert!( + expiration <= from_now(3 * DAY), + "an expiration within the ceiling must be kept, got {expiration}" + ); + + // The settings UI stores an integer, but the YAML instance config and config sync can write + // the same whole number as a string or as `5.0`. Reading either as "unset" would silently + // drop the ceiling, while the token form (`parseMaxTokenExpirationDays`) still showed it. + for (stored, label) in [ + (json!("5"), "string setting"), + (json!(5.0), "float setting"), + ] { + set_max(&db, stored).await; + let resp = create_token(port, json!({ "label": label })).await; + assert_eq!(resp.status(), 201); + let expiration = stored_expiration(&db, label).await; + assert!( + expiration.is_some_and(|e| e > from_now(4 * DAY) && e <= from_now(5 * DAY)), + "{label}: expected the 5 day ceiling, got {expiration:?}" + ); + } + + // A superadmin impersonating a user picks the lifetime too, so the ceiling applies there; + // left out, it would be the one way to mint a token that never expires. + let resp = client() + .post(format!( + "http://localhost:{port}/api/users/tokens/impersonate" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "label": "impersonated", "impersonate_email": "test3@windmill.dev" })) + .send() + .await?; + assert_eq!(resp.status(), 201); + assert!( + stored_expiration(&db, "impersonated").await.is_some(), + "an impersonation token asking for no expiration gets the ceiling" + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_service_accounts_are_exempt(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + set_max(&db, json!(7)).await; + // The same email is a service account in one workspace and an ordinary user in another. + sqlx::query( + "UPDATE usr SET is_service_account = true + WHERE email = 'test2@windmill.dev' AND workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('other', 'other', 'test-user')") + .execute(&db) + .await?; + sqlx::query("INSERT INTO workspace_settings (workspace_id) VALUES ('other')") + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, role) + VALUES ('other', 'test2@windmill.dev', 'test-user-2', false, 'User')", + ) + .execute(&db) + .await?; + + // The token form decides the exemption from this list, so it must carry each membership's flag. + let memberships: serde_json::Value = client() + .get(format!("http://localhost:{port}/api/workspaces/users")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .send() + .await? + .json() + .await?; + for (workspace, is_service_account) in [("test-workspace", true), ("other", false)] { + let membership = memberships["workspaces"] + .as_array() + .and_then(|ws| ws.iter().find(|w| w["id"] == workspace)) + .unwrap_or_else(|| panic!("{workspace} missing from {memberships}")); + assert_eq!(membership["is_service_account"], json!(is_service_account)); + } + + for (label, workspace_id, exempt) in [ + ("own workspace", Some("test-workspace"), true), + ("other workspace", Some("other"), false), + // A workspace-less token has no workspace to match, so a service account anywhere counts. + ("global", None, true), + ] { + let resp = create_token( + port, + json!({ "label": label, "workspace_id": workspace_id }), + ) + .await; + assert_eq!(resp.status(), 201); + assert_eq!( + stored_expiration(&db, label).await.is_none(), + exempt, + "{label}: expected exempt = {exempt}" + ); + } + + // Impersonation checks the impersonated account, not the superadmin minting the token. + let resp = client() + .post(format!( + "http://localhost:{port}/api/users/tokens/impersonate" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "label": "impersonated service account", "impersonate_email": "test2@windmill.dev" })) + .send() + .await?; + assert_eq!(resp.status(), 201); + assert_eq!( + stored_expiration(&db, "impersonated service account").await, + None + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs new file mode 100644 index 0000000000..25942e845c --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs @@ -0,0 +1,193 @@ +//! Request headers reaching a runnable's preprocessor over MCP. +//! +//! The property this pins is structural rather than a filter: the model writes +//! the tool's arguments, which become `event.body`, while the server writes +//! `event.headers`. A model that guesses a header's name can only ever land in +//! `body`, so an identity read from `headers` is one prompt injection cannot +//! forge. Nothing else in the suite exercises MCP argument shaping end to end. +//! +//! Requires: bun runtime, live database (migrations applied by sqlx::test). +#![cfg(feature = "mcp")] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const SCRIPT_PATH: &str = "u/test-user/mcp_hdr_probe"; + +/// A bun lock the executor accepts without installing anything: no dependencies +/// in the `package.json` half, `` for the `bun.lock` half. The empty +/// string is not a substitute: a lock carrying no `//bun.lock` separator is +/// rejected at run time. +const EMPTY_BUN_LOCK: &str = "{}\n//bun.lock\n"; + +/// Echoes the two halves of the event separately, so the assertions can tell +/// which one a value arrived in. +const PREPROCESSOR_SCRIPT: &str = r#" +export async function preprocessor(event: any) { + return { + kind: event.kind, + from_headers: event.headers?.["x-user-id"] ?? "", + from_body: event.body?.x_user_id ?? "", + header_names: Object.keys(event.headers ?? {}).sort(), + }; +} + +export async function main(kind: string, from_headers: string, from_body: string, header_names: string[]) { + return { kind, from_headers, from_body, header_names }; +} +"#; + +async fn insert_mcp_token(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('MCP_TOKEN'::bytea), 'hex'), 'MCP_TOK', 'MCP_TOKEN', 'test@windmill.dev', 'mcp token', true, ARRAY['mcp:all'])", + ) + .execute(db) + .await?; + Ok(()) +} + +/// POST one JSON-RPC message. The endpoint answers either `application/json` or +/// a single-event SSE stream, so strip the `data: ` framing before parsing. +async fn mcp_post(port: u16, headers: &[(&str, &str)], body: Value) -> anyhow::Result { + let mut req = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/mcp/w/test-workspace/mcp" + )) + .header("Authorization", "Bearer MCP_TOKEN") + .header("Accept", "application/json, text/event-stream") + .json(&body); + for (k, v) in headers { + req = req.header(*k, *v); + } + let text = req.send().await?.text().await?; + let payload = text + .lines() + .find_map(|l| l.strip_prefix("data: ")) + .unwrap_or(text.trim()); + serde_json::from_str(payload).map_err(|e| anyhow::anyhow!("unparseable MCP body {text:?}: {e}")) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_preprocessor_receives_the_callers_headers( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + insert_mcp_token(&db).await?; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": SCRIPT_PATH, + "summary": "mcp header probe", + "description": "", + "content": PREPROCESSOR_SCRIPT, + "language": "bun", + "lock": EMPTY_BUN_LOCK, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { "x_user_id": { "type": "string" } }, + "required": [] + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 201, + "create script: {}", + resp.text().await.unwrap_or_default() + ); + + // A supplied lock queues no dependency job, so the version is deployed (hence + // listable and runnable) as soon as the create returns. + let queued: i64 = sqlx::query_scalar( + "SELECT count(*) FROM v2_job_queue WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(queued, 0, "the supplied lock must queue no dependency job"); + + let tools = mcp_post( + port, + &[], + json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}), + ) + .await?; + let tool_name = tools["result"]["tools"] + .as_array() + .and_then(|list| { + list.iter() + .filter_map(|t| t["name"].as_str()) + .find(|n| n.contains("mcp__hdr__probe")) + }) + .ok_or_else(|| anyhow::anyhow!("the deployed script was not listed as a tool: {tools}"))? + .to_string(); + + let result = in_test_worker( + db.clone(), + async { + mcp_post( + port, + // Every name the withheld list covers has to be on the wire, or + // asserting its absence proves nothing. `Authorization` is already + // set by `mcp_post`, and `extract_token` reads it before the + // cookie, so sending one does not disturb auth. + &[ + ("X-User-Id", "alice@corp.example"), + ("Cookie", "session=secret"), + ("Proxy-Authorization", "Basic Zm9v"), + ], + json!({ + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + // The model names the header it wants to spoof. Its value is an + // argument, so it can only ever reach `event.body`. + "params": { "name": tool_name, "arguments": { "x_user_id": "attacker@evil.test" } } + }), + ) + .await + }, + port, + ) + .await?; + + let text = result["result"]["content"][0]["text"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("tool call returned no text content: {result}"))?; + let out: Value = serde_json::from_str(text)?; + + assert_eq!(out["kind"], "mcp", "preprocessor event kind: {out}"); + assert_eq!( + out["from_headers"], "alice@corp.example", + "the caller's header must reach event.headers: {out}" + ); + assert_eq!( + out["from_body"], "attacker@evil.test", + "the model's argument must land in event.body, not overwrite the header: {out}" + ); + + let names: Vec<&str> = out["header_names"] + .as_array() + .map(|a| a.iter().filter_map(|v| v.as_str()).collect()) + .unwrap_or_default(); + assert!( + names.contains(&"x-user-id"), + "event.headers must carry the request's own headers: {names:?}" + ); + for withheld in ["authorization", "cookie", "proxy-authorization"] { + assert!( + !names.contains(&withheld), + "{withheld} is withheld from a preprocessor: {names:?}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/mcp_token_query_param.rs b/backend/windmill-api-integration-tests/tests/mcp_token_query_param.rs new file mode 100644 index 0000000000..d0d6e070f6 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/mcp_token_query_param.rs @@ -0,0 +1,102 @@ +//! The `mcp_disable_token_query_param` switch closes the URL-borne credential path. +//! +//! The rejection is a middleware layered between the `WWW-Authenticate` decorator and +//! everything that reads a token, on both the workspaced and the gateway mount. Each half of +//! that sandwich is pinned: the `WWW-Authenticate` header on the refusal catches the layer +//! being moved outward (a client would lose the pointer that starts OAuth discovery), and +//! refusing a token that was never valid catches it being moved inward past authentication +//! (the URL-borne token would be hashed and looked up before anything refused it). +#![cfg(feature = "mcp")] + +use std::sync::atomic::Ordering; + +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_common::global_settings::MCP_DISABLE_TOKEN_QUERY_PARAM; +use windmill_test_utils::*; + +/// Workspace-less with an `mcp:` scope, which is what the gateway mount requires; the +/// workspaced mount takes its workspace from the path, so one token reaches both. +async fn insert_mcp_token(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('MCP_TOKEN'::bytea), 'hex'), 'MCP_TOK', 'MCP_TOKEN', 'test@windmill.dev', 'mcp token', true, ARRAY['mcp:all'])", + ) + .execute(db) + .await?; + Ok(()) +} + +/// A token that is not in `token` at all. Authentication would refuse it on its own, so a +/// refusal carrying the middleware's own wording is evidence nothing looked it up first. +const BOGUS_TOKEN: &str = "NOT_A_REAL_TOKEN"; + +async fn tools_list(url: &str) -> anyhow::Result { + Ok(reqwest::Client::new() + .post(url) + .header("Accept", "application/json, text/event-stream") + .json(&json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {} })) + .send() + .await?) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_token_query_param_switch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + insert_mcp_token(&db).await?; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + let workspaced = + format!("http://localhost:{port}/api/mcp/w/test-workspace/mcp?token=MCP_TOKEN"); + let gateway = format!("http://localhost:{port}/api/mcp/gateway?token=MCP_TOKEN"); + + assert_eq!( + tools_list(&workspaced).await?.status(), + 200, + "a URL-borne token is the documented default and must keep working while the switch is off" + ); + assert_eq!(tools_list(&gateway).await?.status(), 200); + + MCP_DISABLE_TOKEN_QUERY_PARAM.store(true, Ordering::Relaxed); + + for url in [&workspaced, &gateway] { + let resp = tools_list(url).await?; + assert_eq!( + resp.status(), + 401, + "{url} still admitted a token in the URL" + ); + // What sends the client into the OAuth flow rather than leaving it stuck on a 401. + assert!( + resp.headers().contains_key("www-authenticate"), + "{url} rejected without pointing at the authorization server" + ); + } + + // Refused before authentication, not after: an invalid token gets the middleware's own + // message rather than the generic 401 that looking it up would produce. + let resp = tools_list(&format!( + "http://localhost:{port}/api/mcp/w/test-workspace/mcp?token={BOGUS_TOKEN}" + )) + .await?; + assert_eq!(resp.status(), 401); + assert!( + resp.text().await?.contains("does not accept a token in the MCP URL"), + "an invalid URL token was answered by authentication, so the token was read before \ + the switch refused it" + ); + + // The header stays open: it is the channel the OAuth flow itself hands tokens over on. + let resp = reqwest::Client::new() + .post(format!("http://localhost:{port}/api/mcp/gateway")) + .header("Accept", "application/json, text/event-stream") + .header("Authorization", "Bearer MCP_TOKEN") + .json(&json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {} })) + .send() + .await?; + assert_eq!(resp.status(), 200); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index fe80e5fdf4..059b3aa2d2 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -18,15 +18,16 @@ use windmill_common::{ variables::{build_crypt, encrypt}, }; use windmill_native_triggers::{ - classify_read_failure, decrypt_oauth_data, delete_native_trigger, - delete_workspace_integration, get_workspace_integration, + classify_read_failure, decrypt_oauth_data, delete_native_trigger, delete_workspace_integration, + get_workspace_integration, github::GitHub, google::{parse_stop_channel_params, should_renew_channel}, - http_error_status, list_native_triggers, map_external_error, + grant_refused, http_error_status, list_native_triggers, map_external_error, + native_trigger_is_enabled, nextcloud::NextCloud, - grant_refused, require_native_integration_use, store_native_trigger, - store_workspace_integration, External, ExternalReadFailure, HttpRequestError, - NativeTriggerConfig, OAuthConfig, ServiceName, + require_native_integration_use, set_native_trigger_enabled, store_native_trigger, + store_workspace_integration, update_native_trigger, External, ExternalReadFailure, + HttpRequestError, NativeTriggerConfig, OAuthConfig, ServiceName, }; // ============================================================================ @@ -63,6 +64,7 @@ fn test_authed() -> ApiAuthed { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -455,6 +457,7 @@ async fn test_delete_integration_full_cascade(db: Pool) -> anyhow::Res &trigger_config, json!({"triggerType": "drive"}), None, + true, ) .await?; @@ -545,6 +548,7 @@ async fn test_cleanup_preserves_triggers(db: Pool) -> anyhow::Result<( &trigger_config, json!({"triggerType": "drive"}), None, + true, ) .await?; @@ -602,6 +606,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< }, json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), None, + true, ) .await?; // An unrelated trigger already sitting on the target path must not be reported as moved. @@ -618,6 +623,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< }, json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), None, + true, ) .await?; @@ -783,7 +789,10 @@ fn test_refresh_failures_blame_only_the_grant_they_refuse() { let ok = Some(StatusCode::OK); assert!(grant_refused(ok, r#"{"error":"bad_refresh_token"}"#)); assert!(grant_refused(ok, r#"{"error":"invalid_grant"}"#)); - assert!(!grant_refused(ok, r#"{"access_token":"t","token_type":"bearer"}"#)); + assert!(!grant_refused( + ok, + r#"{"access_token":"t","token_type":"bearer"}"# + )); } /// A service that is busy or broken has not refused anything, and callers react differently to @@ -825,7 +834,9 @@ fn test_transient_service_failures_are_not_refusals() { body: r#"{"message":"Must have admin rights to Repository."}"#.to_string(), }); assert!( - map_external_error(refused).to_string().contains("admin rights"), + map_external_error(refused) + .to_string() + .contains("admin rights"), "a real 403 keeps its guidance" ); } @@ -849,3 +860,115 @@ fn test_only_service_failures_degrade_the_read() { "a non-provider error must pass through unmapped" ); } + +/// The pause switch a webhook delivery is gated on. A trigger arrives enabled, survives an +/// unrelated edit, and an unknown one reads as enabled so a delivery Windmill cannot place is +/// never silently dropped. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_native_trigger_enabled_toggle(db: Pool) -> anyhow::Result<()> { + insert_test_script(&db, "f/test/handler").await?; + let config = NativeTriggerConfig { + script_path: "f/test/handler".to_string(), + is_flow: false, + webhook_token: "abcdefghij1234567890".to_string(), + }; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &config, + json!({"event": "OCA\\Files\\Event\\LoadAdditionalScriptsEvent"}), + None, + true, + ) + .await?; + + assert!( + native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await?, + "a new trigger fires" + ); + + assert!( + set_native_trigger_enabled( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + false + ) + .await? + ); + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await? + ); + + // Saving a configuration must not resume a trigger someone paused. + update_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-1", + &config, + None, + Some("edited"), + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-1").await?, + "an edit leaves the pause in place" + ); + + // A recreate registers a fresh trigger and must be able to come up already paused, in one + // write, rather than being enabled for as long as it takes a second call to arrive. + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &config, + json!({}), + None, + false, + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-2").await? + ); + + // The conflict branch is a re-registration of a trigger that already exists, so it carries no + // opinion about the pause. + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + "ext-2", + &config, + json!({}), + None, + true, + ) + .await?; + assert!( + !native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "ext-2").await?, + "re-registering leaves the pause in place" + ); + + assert!( + !set_native_trigger_enabled( + &db, + "test-workspace", + ServiceName::Nextcloud, + "unknown", + true + ) + .await?, + "nothing to toggle" + ); + assert!( + native_trigger_is_enabled(&db, "test-workspace", ServiceName::Nextcloud, "unknown").await?, + "a trigger Windmill has no row for is not treated as paused" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/offboarding.rs b/backend/windmill-api-integration-tests/tests/offboarding.rs index 4434e6a2b5..7bdeab924b 100644 --- a/backend/windmill-api-integration-tests/tests/offboarding.rs +++ b/backend/windmill-api-integration-tests/tests/offboarding.rs @@ -599,3 +599,59 @@ async fn test_offboard_invalid_target(db: Pool) -> anyhow::Result<()> Ok(()) } + +/// A legacy member named `group-ops` canonicalizes to `g/ops`, the principal the real `ops` group +/// runs as. Offboarding the member must not hand the group's runnables to the replacement. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_offboard_group_prefixed_member_keeps_group_identities( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::raw_sql( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('ops-bot@windmill.dev', 'x', 'password', false, true, 'Ops bot', 'group-ops'); + INSERT INTO usr(workspace_id, email, username, is_admin, role) + VALUES ('test-workspace', 'ops-bot@windmill.dev', 'group-ops', false, 'User'); + INSERT INTO group_(workspace_id, name, summary) VALUES ('test-workspace', 'ops', ''); + INSERT INTO app(workspace_id, path, summary, policy, versions, extra_perms) + VALUES ('test-workspace', 'f/shared/ops_app', '', + '{\"execution_mode\": \"publisher\", \"on_behalf_of\": \"g/ops\", + \"on_behalf_of_email\": \"group-ops@windmill.dev\"}', '{}', '{}');", + ) + .execute(&db) + .await?; + + let preview: serde_json::Value = + authed(client().get(ws_url(port, "offboard_preview/group-ops"))) + .send() + .await? + .json() + .await?; + assert!( + preview["executing_on_behalf"]["apps"].is_null(), + "the group's apps are not the member's to reassign: {preview}" + ); + + let resp = authed(client().post(ws_url(port, "offboard/group-ops"))) + .json(&json!({ + "reassign_to": "u/test-user", + "new_on_behalf_of_user": "test-user", + "delete_user": false + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let principal: Option = sqlx::query_scalar( + "SELECT policy->>'on_behalf_of' FROM app + WHERE workspace_id = 'test-workspace' AND path = 'f/shared/ops_app'", + ) + .fetch_one(&db) + .await?; + assert_eq!(principal.as_deref(), Some("g/ops")); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/protection_rules.rs b/backend/windmill-api-integration-tests/tests/protection_rules.rs index 9469554999..e032acef7b 100644 --- a/backend/windmill-api-integration-tests/tests/protection_rules.rs +++ b/backend/windmill-api-integration-tests/tests/protection_rules.rs @@ -123,6 +123,72 @@ async fn test_protection_rules(db: Pool) -> anyhow::Result<()> { .await?; assert!(!resp.status().is_success(), "Non-admin should be blocked from flows: {}", resp.status()); + // ======================================== + // 4b. ...but a draft-only resource stays deletable: nothing is deployed at + // its path, so its DELETE is a draft discard rather than a deployment. + // ======================================== + + let draft_only_path = "u/test-user-2/draft_only_resource"; + let resp = authed( + client().post(format!("{base}/drafts/update/resource/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ "value": { + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + }})) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Non-admin should save a draft: {}", + resp.text().await? + ); + + let resp = authed( + client().delete(format!("{base}/resources/delete/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Draft-only delete should not be gated by the deploy rules: {}", + resp.text().await? + ); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'resource'::DRAFT_KIND", + ) + .bind(draft_only_path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + // The gate itself is still there for a DEPLOYED resource at the same path. + let resp = authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "Non-admin should still be blocked from creating a resource: {}", + resp.status() + ); + // ======================================== // 5. Admin bypasses protection rule // ======================================== diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 3f0b21b184..53648ec759 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -443,6 +443,30 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let body = resp.json::().await?; assert_eq!(body["description"], "Updated type desc"); + // display_name: an update that omits it, as a push from a CLI predating the field does, + // keeps it; an explicit null clears it. + for (update, expected) in [ + ( + json!({"display_name": "New Test Type"}), + json!("New Test Type"), + ), + ( + json!({"description": "Updated type desc"}), + json!("New Test Type"), + ), + (json!({"display_name": null}), serde_json::Value::Null), + ] { + let resp = authed(client().post(resource_url(port, "type/update", "new_test_type"))) + .json(&update) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let resp = authed_get(port, "type/get", "new_test_type").await; + let body = resp.json::().await?; + assert_eq!(body["display_name"], expected); + } + // type/delete let resp = authed(client().delete(resource_url(port, "type/delete", "new_test_type"))) .send() diff --git a/backend/windmill-api-integration-tests/tests/schedules.rs b/backend/windmill-api-integration-tests/tests/schedules.rs index 89ba0ec2cc..881f11ff43 100644 --- a/backend/windmill-api-integration-tests/tests/schedules.rs +++ b/backend/windmill-api-integration-tests/tests/schedules.rs @@ -113,7 +113,9 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { "expected at least 2 schedules, got {}", list.len() ); - assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule")); + assert!(list + .iter() + .any(|s| s["path"] == "u/test-user/test_schedule")); // --- list_with_jobs --- let resp = authed(client().get(format!("{base}/list_with_jobs"))) @@ -125,18 +127,14 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert!(!list.is_empty()); // --- update --- - let resp = authed(client().post(schedule_url( - port, - "update", - "u/test-user/test_schedule", - ))) - .json(&json!({ - "schedule": "0 0 */12 * * *", - "timezone": "Europe/Paris" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(schedule_url(port, "update", "u/test-user/test_schedule"))) + .json(&json!({ + "schedule": "0 0 */12 * * *", + "timezone": "Europe/Paris" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); // verify update @@ -204,14 +202,11 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(resp.status(), 200); // --- delete --- - let resp = authed(client().delete(schedule_url( - port, - "delete", - "u/test-user/another_schedule", - ))) - .send() - .await - .unwrap(); + let resp = + authed(client().delete(schedule_url(port, "delete", "u/test-user/another_schedule"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "exists", "u/test-user/another_schedule").await; @@ -220,17 +215,108 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Global endpoints ===== // --- preview --- - let resp = authed(client().post(format!( - "http://localhost:{port}/api/schedules/preview" - ))) - .json(&json!({ - "schedule": "0 0 */6 * * *", - "timezone": "UTC" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("http://localhost:{port}/api/schedules/preview"))) + .json(&json!({ + "schedule": "0 0 */6 * * *", + "timezone": "UTC" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?); Ok(()) } + +/// A schedule with no `schedule` row is listed from the `draft` table, so its +/// DELETE drops that draft, then 404s once nothing is left at the path. A legacy +/// (`email IS NULL`) draft is owned by nobody and stays put. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_draft_only_schedule(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/draft_only_schedule"; + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/trigger_schedule/{path}" + ))) + .json(&json!({ "value": { + "path": path, + "schedule": "0 0 */6 * * *", + "timezone": "UTC", + "script_path": "u/test-user/never_deployed", + "is_flow": false, + }})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "save draft: {}", resp.text().await?); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/schedules/list?include_draft_only=true" + ))) + .send() + .await + .unwrap(); + let listed: Vec = resp.json().await?; + assert!( + listed + .iter() + .any(|s| s["path"] == path && s["draft_only"] == json!(true)), + "draft-only schedule should be listed: {listed:?}" + ); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "nothing left at the path"); + + // A legacy (email IS NULL) draft is owned by nobody and keeps the write gate + // on the drafts routes, so this one must not become a second door to it. + let legacy_path = "u/test-user/legacy_draft_only_schedule"; + sqlx::query( + "INSERT INTO draft (workspace_id, email, path, typ, value) \ + VALUES ('test-workspace', NULL, $1, 'trigger_schedule'::DRAFT_KIND, '{}'::json)", + ) + .bind(legacy_path) + .execute(&db) + .await?; + + let resp = authed(client().delete(schedule_url(port, "delete", legacy_path))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 404, + "legacy draft is not this route's to delete" + ); + + let legacy_remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(legacy_path) + .fetch_one(&db) + .await?; + assert_eq!(legacy_remaining, 1, "the legacy draft should survive"); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index 8c88c53454..3a146add27 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -38,6 +38,85 @@ fn new_script(path: &str, summary: &str, content: &str) -> serde_json::Value { }) } +/// A supplied lock queues no dependency job, so if the create does not record its hash nothing +/// ever will, and every importer of this script relocks on each of its deploys forever after. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_script_persists_supplied_lock_hash(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/supplied_lock"; + let lock = r#"{"version":"4","remote":{}}"#; + let mut script = new_script( + path, + "Supplied lock", + "export async function main() { return 42; }", + ); + script["lock"] = json!(lock); + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + let stored_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(stored_hash, windmill_common::scripts::hash_script(lock)); + + // A script deployed before the create recorded hashes has no row, and pushing it unchanged + // creates no version to hang one off. Without the write on that path it would keep its + // importers relocking until someone edited it. + sqlx::query!( + "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .execute(&db) + .await?; + + // The no-op comparison covers every field, so the push has to carry what the first deploy + // filled in by itself; `auto_parent` both resolves the parent and keeps the hash distinct. + script["auto_parent"] = json!(true); + script["ws_error_handler_muted"] = json!(false); + script["assets"] = json!([]); + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create?skip_if_noop=true" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "no-op push: {}", resp.text().await?); + + let versions: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await? + .unwrap_or_default(); + assert_eq!(versions, 1, "no-op push must not create a version"); + + let repaired_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(repaired_hash, windmill_common::scripts::hash_script(lock)); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -797,10 +876,12 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( // What a deploy leaves behind: the old head archived, a new one live at the path. // Copied through a temp table so this does not have to restate every column. - sqlx::query("CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1") - .bind(head) - .execute(&mut *winner) - .await?; + sqlx::query( + "CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1", + ) + .bind(head) + .execute(&mut *winner) + .await?; sqlx::query("UPDATE superseding SET hash = $1, archived = false, parent_hashes = ARRAY[$2]") .bind(head + 1) .bind(head) @@ -818,7 +899,10 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( let resp = tokio::time::timeout(std::time::Duration::from_secs(20), update).await??; let status = resp.status(); let body = resp.text().await?; - assert_eq!(status, 400, "losing the race should not read as success: {body}"); + assert_eq!( + status, 400, + "losing the race should not read as success: {body}" + ); assert!( body.contains("deployed to concurrently"), "the loser must say it was superseded, not that the script is missing: {body}" diff --git a/backend/windmill-api-integration-tests/tests/token_expiry_warning.rs b/backend/windmill-api-integration-tests/tests/token_expiry_warning.rs new file mode 100644 index 0000000000..667e064cca --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/token_expiry_warning.rs @@ -0,0 +1,49 @@ +//! Which user tokens get an "expiring soon" warning queued when they are created. + +use serde_json::json; +use sqlx::types::chrono::Utc; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const DAY: u64 = 24 * 60 * 60; + +async fn warning_queued(db: &Pool, label: &str) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM token_expiry_notification n + JOIN token t ON t.token_hash = n.token_hash WHERE t.label = $1)", + ) + .bind(label) + .fetch_one(db) + .await + .unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_short_lived_tokens_get_no_expiry_warning(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + for (label, lifetime) in [("short", DAY), ("long", 30 * DAY)] { + let resp = reqwest::Client::new() + .post(format!("http://localhost:{port}/api/users/tokens/create")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .json(&json!({ + "label": label, + "expiration": Utc::now() + std::time::Duration::from_secs(lifetime), + })) + .send() + .await?; + assert_eq!(resp.status(), 201); + } + + assert!( + !warning_queued(&db, "short").await, + "a token whose whole lifetime fits in the warning window must not be warned about" + ); + assert!( + warning_queued(&db, "long").await, + "a longer-lived token still gets its warning" + ); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/token_label_idor.rs b/backend/windmill-api-integration-tests/tests/token_label_idor.rs index ddec1a481a..e690bf673f 100644 --- a/backend/windmill-api-integration-tests/tests/token_label_idor.rs +++ b/backend/windmill-api-integration-tests/tests/token_label_idor.rs @@ -179,9 +179,11 @@ async fn test_trigger_token_labels_still_creatable(db: Pool) -> anyhow "http-test-user-2-cd34", "email-test-user-2-ef56", "my-ci-token", - // Minted client-side by the editor (every TypeScript editor load) and the debugger. + // Minted client-side by the editor (every TypeScript editor load), the debugger and + // the object-storage "Test from a worker" button. "Ephemeral lsp token", "debugger-token", + "ephemeral-test-connection: s3_bucket", ] { let resp = create_token_with_label(port, "SECRET_TOKEN_2", label).await; assert_eq!( diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index fb64d9eb77..65d44509d7 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -308,14 +308,17 @@ async fn test_user_endpoints(db: Pool) -> anyhow::Result<()> { let auth_base = format!("http://localhost:{port}/api/auth"); // --- login (will fail: password hash in fixture is fake) --- + // An unparseable stored hash must read as a failed login, not as a server error + // relaying the hash parser's message to an unauthenticated caller. let resp = client() .post(format!("{auth_base}/login")) .json(&json!({"email": "test@windmill.dev", "password": "wrong-password"})) .send() .await .unwrap(); - assert!( - resp.status() == 400 || resp.status() == 401 || resp.status() == 500, + assert_eq!( + resp.status(), + 400, "login: unexpected status {}", resp.status() ); @@ -804,12 +807,16 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a let server = ApiServer::start(db.clone()).await?; let global_base = format!("http://localhost:{}/api/users", server.addr.port()); - sqlx::query!("UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; - sqlx::query!("UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; + sqlx::query!( + "UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; + sqlx::query!( + "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; sqlx::query!( "INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')" ) @@ -910,3 +917,79 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a Ok(()) } + +/// An address with no `password` row can own a draft, and the account paths carry the delete and +/// rename that no foreign key does any more. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_drafts_follow_their_owner_without_a_fkey(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/users"); + + // The destination of the rename below already holds a draft of the same item — it belongs to + // an accountless principal, so `change_email`'s "address is free" check does not see it. + sqlx::query!( + "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'), + ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')" + ) + .execute(&db) + .await?; + + // A null username is how the legacy workspace-level row is encoded, so an owner nobody can + // name must be absent from the owner circles rather than pose as one. + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list?all_users=true" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let listed = resp.json::().await?; + let ext = listed + .as_array() + .unwrap() + .iter() + .find(|d| d["path"] == "u/ext/s") + .expect("the accountless owner's draft is listed"); + assert_eq!(ext.get("draft_users"), None); + + let resp = authed(client().post(format!("{global_base}/change_email/test2@windmill.dev"))) + .json(&json!({ "new_email": "renamed@windmill.dev" })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "change_email: {}", resp.text().await?); + let moved = sqlx::query!( + "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'" + ) + .fetch_all(&db) + .await?; + assert_eq!( + moved + .iter() + .map(|r| (r.email.as_deref(), r.summary.as_deref())) + .collect::>(), + vec![(Some("renamed@windmill.dev"), Some("moving"))], + "the moving account's draft wins the unique index it now collides on" + ); + + let resp = authed(client().delete(format!("{global_base}/delete/test3@windmill.dev"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete_user: {}", resp.text().await?); + let remaining = sqlx::query_scalar!("SELECT path FROM draft ORDER BY path") + .fetch_all(&db) + .await?; + assert_eq!( + remaining, + vec!["u/ext/s".to_string(), "u/two/s".to_string()], + "the deleted account's draft goes, the accountless owner's stays" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs index 4e25be1fa5..dc8e8dcc00 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -923,6 +923,7 @@ async fn test_pull_stays_on_the_workspace_lane(db: Pool) -> anyhow::Re &db, "test-workspace", &repo, + ("test-user", "test@windmill.dev"), None, false, None, diff --git a/backend/windmill-api-integration-tests/tests/workspace_encryption_key_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_encryption_key_git_sync.rs index 8edfd39d61..a22ad974c3 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_encryption_key_git_sync.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_encryption_key_git_sync.rs @@ -275,6 +275,67 @@ async fn test_encryption_key_rotation_dispatches_batched_git_sync( Ok(()) } +/// Stored repository tokens and webhook secrets are encrypted under the +/// workspace key but never synced, so a rotation has to carry them over even +/// when the caller skips re-encrypting variables. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_encryption_key_rotation_reencrypts_git_sync_secrets( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::variables::{build_crypt, crypt_from_key_with_suffix, decrypt, encrypt}; + initialize_tracing().await; + + create_folder(&db, "28103").await?; + create_git_repo_resource(&db).await?; + let sync_script_path = "f/28103/test_sync_script_git_secrets"; + create_sync_script(&db, sync_script_path).await?; + setup_git_sync_config(&db, sync_script_path).await?; + + let mc = build_crypt(&db, "test-workspace").await?; + sqlx::query( + r#" + UPDATE workspace_settings SET + git_credentials = jsonb_build_array(jsonb_build_object( + 'token', $1::text, 'repo_identity', 'https://gitlab.example.com/grp/proj')), + git_sync = jsonb_set(git_sync, '{repositories,0,auto_pull}', jsonb_build_object( + 'enabled', true, 'mode', 'webhook', 'webhook_id', 1, 'webhook_secret', $2::text)) + WHERE workspace_id = 'test-workspace' + "#, + ) + .bind(encrypt(&mc, "stored-token")) + .bind(encrypt(&mc, "hook-secret")) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let new_key = "c".repeat(64); + let resp = authed(client().post(format!("{base}/encryption_key"))) + .json(&json!({"new_key": new_key, "skip_reencrypt": true})) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "set_encryption_key failed: {}", + resp.text().await? + ); + + let (token, secret): (String, String) = sqlx::query_as( + "SELECT git_credentials->0->>'token', git_sync#>>'{repositories,0,auto_pull,webhook_secret}' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + let new_mc = crypt_from_key_with_suffix(&new_key, ""); + assert_eq!(decrypt(&new_mc, token)?, "stored-token"); + assert_eq!(decrypt(&new_mc, secret)?, "hook-secret"); + + Ok(()) +} + /// Regression test for the non-debouncing fallback: a workspace whose sync /// script predates hub version 28103 must still receive git-sync jobs for the /// encryption_key entry and every re-encrypted secret. Before the fallback was diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 65fe4241cc..ff603269e7 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -889,6 +889,53 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +/// A workspace with no provider of its own is served the instance config, but the +/// `copilot_disabled` flag must still come from the workspace's own row. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_get_copilot_info_keeps_workspace_copilot_disabled_over_instance_fallback( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + sqlx::query("UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2") + .bind(json!({ "copilot_disabled": true })) + .bind("test-workspace") + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind("ai_config") + .bind(json!({ + "providers": { + "openai": { + "resource_path": "u/test-user/openai_instance", + "models": ["gpt-4o-mini"] + } + } + })) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{base}/get_copilot_info"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert_eq!( + settings["providers"]["openai"]["models"][0], "gpt-4o-mini", + "instance providers are still served" + ); + assert_eq!(settings["copilot_disabled"], true); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -941,7 +988,12 @@ async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyh .send() .await .unwrap(); - assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert_eq!( + resp.status(), + 200, + "disable on fork: {}", + resp.text().await? + ); assert!(!stored().await?); Ok(()) @@ -1044,9 +1096,11 @@ async fn test_create_service_account_drops_orphaned_group_memberships( .await?; // Same username, different workspace, and very much alive — must not be touched. - sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')") - .execute(&db) - .await?; + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')", + ) + .execute(&db) + .await?; sqlx::query( "INSERT INTO group_ (workspace_id, name, summary) VALUES ('other-workspace', 'all', 'All users'), @@ -1112,3 +1166,47 @@ async fn test_create_service_account_drops_orphaned_group_memberships( Ok(()) } + +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_edit_auto_invite_preserves_instance_groups(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query( + r#"UPDATE workspace_settings + SET auto_invite = '{"instance_groups": ["eng"], "instance_groups_roles": {"eng": "developer"}}' + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + // enable, then disable + for body in [ + json!({"operator": false, "invite_all": true, "auto_add": false}), + json!({}), + ] { + let resp = authed(client().post(format!("{base}/edit_auto_invite"))) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{body}: {}", resp.text().await?); + + let auto_invite: serde_json::Value = sqlx::query_scalar( + "SELECT auto_invite FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(auto_invite["instance_groups"], json!(["eng"]), "{body}"); + assert_eq!( + auto_invite["instance_groups_roles"], + json!({"eng": "developer"}), + "{body}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index fbfd656cdf..7184f8ecec 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -26,7 +26,9 @@ use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed}; use windmill_common::{ db::{UserDB, UserDbWithAuthed}, error::{self, Error}, - flow_conversations::{add_message_to_conversation_tx, MessageType}, + flow_conversations::{ + add_message_to_conversation_tx, message_attachments, MessageExtras, MessageType, + }, get_latest_flow_version_info_for_path, jobs::{ check_tag_available_for_workspace_internal, format_result, script_path_to_payload, @@ -38,8 +40,8 @@ use windmill_common::{ FlowVersionInfo, DB, }; use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned, - PushIsolationLevel, + cancel_job, get_result_and_success_by_id_from_flow, parse_result_object, push, PushArgs, + PushArgsOwned, PushIsolationLevel, }; use crate::types::RunJobQuery; @@ -374,9 +376,9 @@ pub async fn run_wait_result_internal( } pub fn result_to_response(result: Box, success: bool) -> error::Result { - let composite_result = serde_json::from_str::(result.get()); + let composite_result = parse_result_object::(result.get()); match composite_result { - Ok(WindmillCompositeResult { + Some(WindmillCompositeResult { windmill_status_code, windmill_content_type, windmill_headers, @@ -416,11 +418,31 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result let mut headers = HeaderMap::new(); + // A reverse proxy consumes hop-by-hop headers instead of forwarding them and + // drops every header named by `Connection`, so a script could use one to strip + // the sandbox headers this function adds before they reach the browser. + const HOP_BY_HOP_HEADERS: [&str; 9] = [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ]; + if let Some(windmill_headers) = windmill_headers { for (k, v) in windmill_headers { let k = HeaderName::from_str(k.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header name {k}: {err}")) })?; + if HOP_BY_HOP_HEADERS.contains(&k.as_str()) { + return Err(Error::ExecutionErr(format!( + "windmill_headers cannot set the hop-by-hop header \"{k}\"" + ))); + } let v = HeaderValue::from_str(v.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header value {v}: {err}")) })?; @@ -428,6 +450,22 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result } } + // The script controls the content type and body, and run_wait_result and sync + // HTTP routes are reachable by top-level GET navigation with the session cookie: + // sandbox the document into an opaque origin so HTML can never run with the + // viewer's session. Inserted after `wm_headers` so a script cannot override it. + headers.insert( + http::header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + headers.insert( + http::header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static( + "sandbox allow-scripts allow-forms allow-popups \ + allow-popups-to-escape-sandbox allow-downloads allow-modals", + ), + ); + if let Some(content_type) = windmill_content_type { let serialized_json_result = result_value .map(|val| val.get().to_owned()) @@ -617,9 +655,11 @@ pub async fn set_flow_memory_id( pub async fn process_flow_run_query_params( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, job_id: Uuid, + w_id: &str, + flow_path: &str, run_query: &RunJobQuery, ) -> error::Result<()> { - if let Some(memory_id) = run_query.memory_id { + if let Some(memory_id) = run_query.memory_key(w_id, flow_path) { set_flow_memory_id(tx, job_id, memory_id).await?; } Ok(()) @@ -632,10 +672,19 @@ pub async fn handle_chat_conversation_messages( flow_path: &str, run_query: &RunJobQuery, user_message_raw: Option<&Box>, + job_id: Uuid, + is_test: bool, + // The run's args, for the files the message carried. + args: &HashMap>, ) -> error::Result<()> { - let memory_id = run_query.memory_id.ok_or_else(|| { + // Names the query parameter rather than the field: it is not a flow argument, and + // supplying it as one is the first thing tried on reading `memory_id is required`. + let memory_id = run_query.memory_key(w_id, flow_path).ok_or_else(|| { windmill_common::error::Error::BadRequest( - "memory_id is required for chat-enabled flows".to_string(), + "memory_id is required for chat-enabled flows. Pass it as the `memory_id` query \ + parameter, not as a flow argument: it names the conversation the turn belongs to, \ + so a fresh UUID starts one and reusing a UUID continues it." + .to_string(), ) })?; @@ -659,17 +708,22 @@ pub async fn handle_chat_conversation_messages( &authed.username, &user_message, memory_id, + is_test, ) .await?; + // The run this message started. The row keeps the files the message carried as + // references; its args are the only record of every other flow input, and nothing + // written later points at them: an assistant row holds the AI agent step's job. add_message_to_conversation_tx( tx, memory_id, - None, + Some(job_id), &user_message, MessageType::User, None, true, + Some(&MessageExtras { attachments: message_attachments(args), ..Default::default() }), ) .await?; @@ -777,7 +831,7 @@ pub async fn run_flow<'c>( .await?; // Set memory_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { + if let Some(memory_id) = run_query.memory_key(w_id, flow_path) { set_flow_memory_id(&mut tx, uuid, memory_id).await?; } @@ -790,6 +844,9 @@ pub async fn run_flow<'c>( &flow_path.to_string(), &run_query, args.args.get("user_message"), + uuid, + false, + &args.args, ) .await?; } @@ -1104,6 +1161,65 @@ mod result_to_response_tests { resp.headers().get(http::header::CONTENT_TYPE).unwrap(), "text/html" ); + assert_sandboxed(resp.headers()); assert_eq!(body_bytes(resp).await, b"

hi

"); } + + fn assert_sandboxed(headers: &HeaderMap) { + assert_eq!( + headers.get(http::header::X_CONTENT_TYPE_OPTIONS).unwrap(), + "nosniff" + ); + let csp = headers + .get(http::header::CONTENT_SECURITY_POLICY) + .expect("content-security-policy") + .to_str() + .unwrap(); + assert!(csp.starts_with("sandbox "), "csp: {csp}"); + assert!(!csp.contains("allow-same-origin"), "csp: {csp}"); + } + + #[tokio::test] + async fn custom_headers_cannot_override_sandbox() { + // wm_headers is script-controlled: a content-type set there replaces the JSON + // one even without wm_content_type, and the sandbox headers must survive an + // attempt to override them. + let resp = result_to_response( + raw( + r#"{"wm_headers":{"content-type":"text/html","content-security-policy":"default-src *","x-content-type-options":"none"},"result":"

hi

"}"#, + ), + true, + ) + .expect("response"); + + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html" + ); + assert_sandboxed(resp.headers()); + } + + #[tokio::test] + async fn hop_by_hop_custom_headers_are_rejected() { + // A proxy drops every header named by `Connection`, which would strip the + // sandbox headers on the way to the browser. + for name in ["connection", "Connection", "transfer-encoding", "upgrade"] { + let res = result_to_response( + raw(&format!( + r#"{{"wm_content_type":"text/html","wm_headers":{{"{name}":"content-security-policy, x-content-type-options"}},"result":"

hi

"}}"# + )), + true, + ); + assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); + } + } + + #[tokio::test] + async fn array_result_is_not_a_composite_response() { + let json = r#"[201,"text/html",null,null,"

hi

"]"#; + let resp = result_to_response(raw(json), true).expect("response"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(body_bytes(resp).await, json.as_bytes()); + } } diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index 05f5a51bbc..6389134031 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -692,16 +692,64 @@ pub async fn delete_jobs( .await? .rows_affected(); - let conversation_message_deleted = sqlx::query!( + // One row per message deleted, so the conversation of a chat losing several appears + // several times: the count is taken before the dedup below. + let mut conversation_ids: Vec = sqlx::query_scalar!( "DELETE FROM flow_conversation_message m USING flow_conversation c - WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)", + WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2) + RETURNING m.conversation_id", &w_id, &job_ids ) - .execute(&mut *tx) - .await? - .rows_affected(); + .fetch_all(&mut *tx) + .await?; + let conversation_message_deleted = conversation_ids.len() as u64; + + // Same rule, lock and statement order as retention (windmill_common::jobs::delete_jobs, + // which says why): a conversation with no messages left goes, and its memory with it. + conversation_ids.sort_unstable(); + conversation_ids.dedup(); + let mut memory_deleted = 0; + let mut conversation_deleted = 0; + if !conversation_ids.is_empty() { + sqlx::query_scalar!( + "SELECT id FROM flow_conversation WHERE id = ANY($1) AND workspace_id = $2 ORDER BY id FOR UPDATE", + &conversation_ids, + &w_id + ) + .fetch_all(&mut *tx) + .await?; + memory_deleted = sqlx::query!( + "DELETE FROM ai_agent_memory a + USING flow_conversation c + WHERE c.id = ANY($1) + AND c.workspace_id = $2 + AND a.conversation_id = c.id + AND a.workspace_id = c.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids, + &w_id + ) + .execute(&mut *tx) + .await? + .rows_affected(); + conversation_deleted = sqlx::query!( + "DELETE FROM flow_conversation c + WHERE c.id = ANY($1) + AND c.workspace_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids, + &w_id + ) + .execute(&mut *tx) + .await? + .rows_affected(); + } // Resolutions are not exported, so a delete-then-reimport of the same UUID would // otherwise resurrect the old annotation on a job that never carried one. @@ -737,6 +785,8 @@ pub async fn delete_jobs( + zombie_deleted + dispatch_event_deleted + conversation_message_deleted + + memory_deleted + + conversation_deleted + resolution_deleted + jobs_deleted; diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index 835c4c7e34..857b2e02e2 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -585,6 +585,8 @@ pub fn list_completed_jobs_query( let mut sqlb = SqlBuilder::select_from("v2_job_completed") .fields(fields) .order_by( + // The runs page picks its pagination cursor column from this same rule + // (frontend/src/lib/components/runs/useJobsLoader.svelte.ts); change both together. if lq.completed_before.is_some() || lq.completed_after.is_some() || lq.success == Some(false) diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index 6c2d776e30..f37040bd11 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -47,13 +47,25 @@ pub struct RunJobQuery { pub cache_ignore_s3_path: Option, pub skip_preprocessor: Option, pub poll_delay_ms: Option, - pub memory_id: Option, + /// Any string; see [`RunJobQuery::memory_key`]. + pub memory_id: Option, pub trigger_external_id: Option, pub service_name: Option, pub suspended_mode: Option, } impl RunJobQuery { + /// The memory id as stored in `flow_status.memory_id`: a uuid is kept, any other string hashed + /// within the workspace and the flow being run. + pub fn memory_key(&self, workspace_id: &str, flow_path: &str) -> Option { + self.memory_id + .as_deref() + .filter(|memory_id| !memory_id.trim().is_empty()) + .map(|memory_id| { + windmill_common::flow_conversations::memory_key(workspace_id, flow_path, memory_id) + }) + } + pub async fn get_scheduled_for( &self, db: &DB, diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 6ae902e984..dfd4a6e3ba 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -31,8 +31,8 @@ use windmill_common::{ self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, }, user_drafts::{ - delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, - UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, fetch_draft_only_list_rows, + overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, }, utils::{ escape_ilike_pattern, not_found_if_none, paginate, Pagination, ScheduleType, StripPath, @@ -332,7 +332,7 @@ async fn create_schedule( ) .await?; // email is still written for backwards compat with old workers that don't know about permissioned_as - let resolved_email = windmill_common::users::get_email_from_permissioned_as( + let resolved_email = windmill_common::users::get_email_from_permissioned_as_uncached( &resolved_permissioned_as, &w_id, &db, @@ -545,18 +545,14 @@ async fn edit_schedule( reject_reserved_schedule_path(path)?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; - let mut tx = user_db.begin(&authed).await?; // Check schedule for error ScheduleType::from_str(&es.schedule, es.cron_version.as_deref(), true)?; - // Validate dynamic_skip if provided - if let Some(handler_path) = &es.dynamic_skip { - validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; - } - let resolved_edited_by = resolve_edited_by(&authed); + // Resolved on the (non-RLS) pool before the RLS transaction opens: the lookup mid-transaction + // would hold a second connection while `tx` is checked out. let resolved_permissioned_as = resolve_permissioned_as( es.permissioned_as.as_ref(), es.preserve_permissioned_as, @@ -568,7 +564,7 @@ async fn edit_schedule( let resolved_email = if resolved_permissioned_as != windmill_common::users::username_to_permissioned_as(&authed.username) { - windmill_common::users::get_email_from_permissioned_as( + windmill_common::users::get_email_from_permissioned_as_uncached( &resolved_permissioned_as, &w_id, &db, @@ -585,6 +581,13 @@ async fn edit_schedule( Some(&resolved_email), )?; + let mut tx = user_db.begin(&authed).await?; + + // Validate dynamic_skip if provided + if let Some(handler_path) = &es.dynamic_skip { + validate_dynamic_skip(&mut tx, &w_id, handler_path).await?; + } + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; let schedule = sqlx::query_as!( @@ -1126,35 +1129,23 @@ pub async fn set_enabled( check_scopes(&authed, || format!("schedules:write:{}", path))?; reject_reserved_schedule_path(path)?; - // Block enabling a schedule in a fork when the parent has the same path - // (regardless of parent's enabled flag), unless force=true. Two enabled - // crons fire in lockstep; even when the parent is currently disabled the - // user is likely to re-enable it later, at which point both fire — better - // to surface that risk at every fork-side enable. There's no namespacing - // fix for schedules (Phase 3 doesn't help cron); the user has to confirm - // or point the script at fork-only side effects. + // Block enabling a schedule in a fork when an ancestor has the same path + // (regardless of its enabled flag), unless force=true. Two enabled crons + // fire in lockstep; even when the ancestor is currently disabled the user + // is likely to re-enable it later, at which point both fire — better to + // surface that risk at every fork-side enable. There's no namespacing fix + // for schedules (Phase 3 doesn't help cron); the user has to confirm or + // point the script at fork-only side effects. if payload.enabled && !payload.force { - let parent_id: Option = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1", - &w_id + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, "schedule", &w_id, path, ) - .fetch_optional(&mut *tx) .await? - .flatten(); - if let Some(parent_id) = parent_id { - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM schedule WHERE workspace_id = $1 AND path = $2)", - &parent_id, - path, - ) - .fetch_one(&mut *tx) - .await?; - if exists == Some(true) { - return Err(Error::BadRequest(format!( - "fork-conflict:schedule:{}", - parent_id - ))); - } + { + return Err(Error::BadRequest(format!( + "fork-conflict:schedule:{}", + ancestor_id + ))); } } let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; @@ -1331,6 +1322,18 @@ async fn delete_schedule( .flatten(); if exists.is_none() { + drop(tx); + if delete_draft_only_for_path( + &db, + &w_id, + UserDraftItemKind::TriggerSchedule, + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only schedule {} deleted", path)); + } return Err(windmill_common::error::Error::NotFound(format!( "Schedule {} not found", path @@ -1684,9 +1687,9 @@ pub use windmill_queue::schedule::clear_schedule; #[derive(Deserialize)] pub struct SetEnabled { pub enabled: bool, - /// Bypass the parent-state warning when enabling a schedule in a fork - /// whose parent has the same path enabled. The frontend sets this after - /// the user confirms the duplicate-firing dialog. + /// Bypass the fork-conflict warning when enabling a schedule in a fork + /// while an ancestor workspace has the same path. The frontend sets this + /// after the user confirms the duplicate-firing dialog. #[serde(default)] pub force: bool, } diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs index 9c503ae1ae..1c0bc6d57f 100644 --- a/backend/windmill-api-scripts/src/asset_inference.rs +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -78,20 +78,26 @@ fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> { | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Nativets - | ScriptLang::Go => Some("//"), + | ScriptLang::Go + | ScriptLang::Php => Some("//"), _ => None, } } /// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): ` /// volume: ` lines in the leading comment block, each an `rw` volume -/// asset. Scanning stops at the first non-comment line (blank lines are -/// skipped), exactly like the frontend. +/// asset. Scanning stops at the first non-comment line; blank lines and PHP's +/// opening tag line are skipped whole (the tag may carry code, so an annotation +/// must sit on its own line below it), exactly like the frontend. fn parse_volume_annotations(content: &str, prefix: &str) -> Vec { let mut volumes = Vec::new(); for line in content.lines() { let trimmed = line.trim(); - if trimmed.is_empty() { + if trimmed.is_empty() + || trimmed + .get(..5) + .is_some_and(|p| p.eq_ignore_ascii_case(" = got.iter().filter(|a| a.kind == AssetKind::Volume).collect(); + assert_eq!(vols.len(), 1); + assert_eq!(vols[0].path, "my_vol"); + } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 79c6c99b9b..51f2619805 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::process_relative_imports; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_common::{ assets::{ @@ -55,7 +55,7 @@ use windmill_common::{ min_version_supports_runnable_settings_v0, RunnableSettings, RunnableSettingsTrait, }, scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, - utils::{paginate_without_limits, WarnAfterExt}, + utils::{paginate_optional, paginate_without_limits, WarnAfterExt}, worker::CLOUD_HOSTED, }; use windmill_object_store::upload_artifact_to_store; @@ -216,12 +216,15 @@ async fn list_scripts( // a member of has no `usr` row, so fall back to their instance-derived username // (`password.username`), or their email when derivation is disabled — this keeps the // raw email out of the payload whenever a derived username exists. The genuine - // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL). + // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL), + // which is why an owner that resolves to no name at all — an external JWT's subject + // has neither row — is dropped: None is read as "legacy" downstream. "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() @@ -1070,6 +1073,65 @@ fn modules_eq( } } +/// Recorded for the empty lock a codebase or a language with no lock generation carries as well as +/// for a real one: the worker writes `hash_script("")` in the same situation, and a path going from +/// a real lock to an empty one has to stop matching what its importers recorded, or they wrongly +/// skip rather than merely relock too often. +fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { + [(path.to_string(), hash_script(lock))] +} + +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1256,6 +1318,47 @@ async fn create_script_internal<'c>( } } + // A retired path keeps its versions, and the newest is where a redeploy belongs: hashed + // as a first deploy instead, unchanged content lands on the row the path's own first + // version already holds. A deleted version still counts — its row keeps the hash it was + // deployed under even once the content is wiped, so skipping it is what collides. + // + // Any parentless deploy, not only an `auto_parent` one: the CLI names no parent for a + // path its listing no longer shows, which is where a retried push lands. Gated on + // nothing being live there, so a parentless deploy onto a live path still meets the + // path conflict the match below raises. + let mut parent_adopted_from_retired_path = false; + if ns.parent_hash.is_none() && clashing_script.is_none() { + // Locked, not merely read: a competing deploy chaining onto this same candidate + // takes `FOR UPDATE` on it before inserting, so holding the row is what serializes + // the two. Probe first and the child still uncommitted reads as absent. + let candidate = sqlx::query_scalar::<_, i64>( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 \ + ORDER BY created_at DESC LIMIT 1 FOR UPDATE", + ) + .bind(&ns.path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + // Adoptable only if nothing already descends from it: a rename leaves its source + // path holding a version whose child lives at the destination, and a second child + // forks a lineage the guard below requires to be linear. Nothing adoptable means a + // fresh lineage, which has no parent to vary its hash and can still collide. + ns.parent_hash = match candidate { + Some(hash) => sqlx::query_scalar!( + "SELECT 1 FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + hash, + &w_id + ) + .fetch_optional(&db) + .await? + .is_none() + .then_some(ScriptHash(hash)), + None => None, + }; + parent_adopted_from_retired_path = ns.parent_hash.is_some(); + } + // Must stay below the parent resolution above: an auto_parent deploy hashed before // it carries a first deploy's lineage, so redeploying content the path has held // before collides with that archived version instead of superseding it. The @@ -1300,20 +1403,42 @@ async fn create_script_internal<'c>( )); }; + // Unscoped, and sound only under the lock above: linearity is a property of the + // lineage, not of what this caller may read. A child can sit where they cannot + // see it — a folder they renamed it into, or grants an adopting deploy reset — + // and asked through `tx` it reads as absent, letting the fork through. let clashing_hash_o = sqlx::query_scalar!( "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", p_hash.0, &w_id ) - .fetch_optional(&mut *tx) + .fetch_optional(&db) .await?; if let Some(clashing_hash) = clashing_hash_o { - return Err(Error::BadRequest(format!( - "A script with hash {} with same parent_hash has been found. However, the \ - lineage must be linear: no 2 scripts can have the same parent", - ScriptHash(clashing_hash) - ))); + // Named only when the caller could already read it. The probe above has to be + // unscoped to be correct, but a hash alone reads a script's content back + // through `raw/h/{hash}`, which authorizes nothing per script — so echoing one + // the caller cannot see hands them a way to fetch it. + let visible_to_caller = sqlx::query_scalar!( + "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", + clashing_hash, + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + return Err(Error::BadRequest(if visible_to_caller { + format!( + "A script with hash {} with same parent_hash has been found. However, \ + the lineage must be linear: no 2 scripts can have the same parent", + ScriptHash(clashing_hash) + ) + } else { + "A script with the same parent_hash has been found. However, the lineage \ + must be linear: no 2 scripts can have the same parent" + .to_owned() + })); }; let ScriptWithStarred { script: ps, .. } = @@ -1337,6 +1462,12 @@ async fn create_script_internal<'c>( parent_hash = %p_hash.0, "Skipping no-op script deploy (identical to parent)" ); + // The version is unchanged, but the row recording its lock's hash may never have + // been written — nothing else writes it for a supplied lock, and a path only ever + // pushed unchanged would otherwise keep its importers relocking forever. + if let Some(lock) = ps.lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } return Ok((p_hash.clone(), tx, None, Vec::new())); } @@ -1370,7 +1501,15 @@ async fn create_script_internal<'c>( } Some(_) | None => Ok(Some(ParentInfo { p_hashes: ph, - perms: ps.extra_perms, + // A version adopted above was taken for its lineage, not its grants: a + // retired path may be reused by a different script, which must not start + // life holding an ACL nobody gave it — including one `delete/h` purged. + // A parent the caller named still carries them, as unarchive expects. + perms: if parent_adopted_from_retired_path { + json!({}) + } else { + ps.extra_perms + }, p_path: ps.path, })), }; @@ -1548,34 +1687,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -1884,6 +2079,13 @@ async fn create_script_internal<'c>( .execute(&mut *tx) .await?; + // A lock that is not left to a dependency job queues none, so this is the only place its hash + // can be recorded. `try_skip_relock` treats a missing hash for an imported script as changed, + // so leaving the row out makes every importer of this path relock on every deploy of it. + if let Some(lock) = lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } + // Update ci_test_reference table for test scripts // Delete by both new and old path to handle renames let old_path = parent_hashes_and_perms.as_ref().map(|x| x.p_path.as_str()); @@ -2110,6 +2312,20 @@ async fn create_script_internal<'c>( .await?; } + if p_path != &ns.path { + // Everything left at the old path is a draft this deploy didn't + // consume — teammates' rows, and the deployer's own when the caller + // asked us to keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::Script], + p_path, + &ns.path, + ) + .await?; + } + sqlx::query!( "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", ns.path, @@ -2187,19 +2403,32 @@ async fn create_script_internal<'c>( tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } - } else if !skip_draft_deletion { - // See the matching branch above — only wipe the deployer's own - // draft (plus the legacy NULL-email row). - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script' \ - AND (email = $3 OR email IS NULL)", - ns.path, - &w_id, - &authed.email, - ) - .execute(&mut *tx) - .await?; + } else { + if !skip_draft_deletion { + // See the matching branch above — only wipe the deployer's own + // draft (plus the legacy NULL-email row). + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script' \ + AND (email = $3 OR email IS NULL)", + ns.path, + &w_id, + &authed.email, + ) + .execute(&mut *tx) + .await?; + } } + // Every deploy, not only a new script: an archived script's draft can be moved away + // (`move_draft` ignores archived rows), and unarchiving redeploys at the same path, + // where a route left behind would send the live script's saves to the moved draft. + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::Script], + &ns.path, + p_path_opt.as_deref(), + ) + .await?; if p_hashes.is_some() && !p_hashes.unwrap().is_empty() { audit_log( &mut *tx, @@ -2323,27 +2552,28 @@ async fn create_script_internal<'c>( // while its own finished runs still render from them. Clearing by path // would empty those run pages for good. if ns.language != ScriptLang::Dbt { - // The saved retry state does go: nothing regenerates it, it is keyed by - // path alone, and it carries one user's failed invocation and its - // arguments. No dbt version is live at this path any more to resume it. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, &ns.path).await?; + // The saved run and environment state do go: nothing regenerates them, + // both are keyed by path alone, and they carry one user's failed + // invocation with its arguments and the project's own manifest. No dbt + // version is live at this path any more to resume or defer to. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, &ns.path).await?; } if let Some(ref old) = p_path_opt { if old != &ns.path { clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; clear_static_asset_usage(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; - // The saved retry state travels rather than being cleared: nothing + // The saved state travels rather than being cleared: nothing // regenerates it, so dropping it would throw away a resumable - // failure for what is only a rename. Only while the destination is - // still dbt — a rename that also converts the language would - // otherwise reinstate at the new path the state the branch above - // just cleared, leaving one user's arguments and results under a - // path no dbt script occupies. + // failure and every deferral until the next full run, for what is + // only a rename. Only while the destination is still dbt — a rename + // that also converts the language would otherwise reinstate at the + // new path the state the branch above just cleared, leaving one + // user's arguments and results under a path no dbt script occupies. if ns.language == ScriptLang::Dbt { - windmill_common::dbt_manifest::move_dbt_run_state(&mut tx, &w_id, old, &ns.path) + windmill_common::dbt_manifest::move_dbt_script_state(&mut tx, &w_id, old, &ns.path) .await?; } else { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, old).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, old).await?; } } } @@ -2351,16 +2581,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only @@ -2831,17 +3088,24 @@ async fn get_script_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + // Unasked-for, this listing stays whole: the deployment-history panels, the restart + // picker and the CLI all read it without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by s.created_at DESC", + ORDER by s.created_at DESC + LIMIT $3 OFFSET $4", w_id, path, + per_page, + offset, ) .fetch_all(&mut *tx) .await?; @@ -2853,6 +3117,7 @@ async fn get_script_history( script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, created_at: Some(row.created_at), + created_by: Some(row.created_by), }) .collect(); return Ok(Json(result)); @@ -2867,7 +3132,7 @@ async fn get_latest_version( check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let row_o = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC LIMIT 1", @@ -2883,6 +3148,7 @@ async fn get_latest_version( script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, created_at: Some(row.created_at), + created_by: Some(row.created_by), }; return Ok(Json(Some(result))); } else { @@ -3554,7 +3820,11 @@ async fn archive_script_by_path( path, &w_id ) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as the by-hash routes are: + // committed on its own, a cleanup that then fails leaves dbt state at a path + // no live version occupies, for whatever is created there next to defer + // through. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; @@ -3562,9 +3832,10 @@ async fn archive_script_by_path( // The graph stays: the pinned read resolves versions through a CTE that // already skips archived rows, so it stops answering for current relations // either way, while deleting it would empty the Models panel of every - // completed run of the project. Retry state does go — nothing may resume a - // script that is no longer live. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + // completed run of the project. The saved run and environment state do go — + // nothing may resume a script that is no longer live, and nothing may defer + // through what it last built. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; // Pipeline event hygiene: an archived script must not be triggered by // anything. Wipe declared `// on ...` edges (asset-event subscribers // look these up). @@ -3649,7 +3920,7 @@ async fn archive_script_by_hash( clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; // The version's graph stays: its finished runs still render from it, and // the live-version CTE already skips archived rows. Deletion clears it. - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3712,7 +3983,12 @@ async fn delete_script_by_hash( ) .bind(&hash.0) .bind(&w_id) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as `archive_script_by_hash` + // already does. Committed on its own, it opens a window where the path has + // no live version and a concurrent deploy can take it — and the retirement + // guard below then finds that new script live, keeps the old project's dbt + // state, and leaves the replacement able to defer through its manifest. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; @@ -3725,7 +4001,7 @@ async fn delete_script_by_hash( windmill_common::dbt_manifest::clear_dbt_manifest_version(&mut tx, &w_id, &script.path, hash.0) .await?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3826,11 +4102,11 @@ async fn delete_script_by_path( // After the DELETE, never before: every dbt writer locks the `script` row // first, so taking a sidecar ahead of it deadlocks one of the pair. The - // VERSIONED graph needs no clear at all, cascading off `script`; the retry - // state does, being keyed by path alone and so inherited by whatever is - // created here next, and so do the editor's own graphs, whose NULL + // VERSIONED graph needs no clear at all, cascading off `script`; the saved + // run and environment state do, being keyed by path alone and so inherited + // by whatever is created here next, and so do the editor's own graphs, whose NULL // `script_hash` satisfies that foreign key without riding its cascade. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, path).await?; if !trash_scripts.is_empty() { @@ -3999,7 +4275,7 @@ async fn delete_scripts_bulk( // Same reason as the single-path delete, over every requested path rather // than the deleted ones: a path that had no script left can still hold state. for p in &request.paths { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, p).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, p).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, p).await?; } diff --git a/backend/windmill-api-settings/Cargo.toml b/backend/windmill-api-settings/Cargo.toml index bc5078bce4..0ec99ef6d0 100644 --- a/backend/windmill-api-settings/Cargo.toml +++ b/backend/windmill-api-settings/Cargo.toml @@ -11,7 +11,7 @@ path = "src/lib.rs" [features] default = [] enterprise = ["license"] -private = ["windmill-common/private"] +private = ["windmill-common/private", "windmill-audit/private"] parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] license = ["dep:rsa"] @@ -19,6 +19,7 @@ license = ["dep:rsa"] windmill-ai = { workspace = true, default-features = false } windmill-alerting.workspace = true windmill-api-auth.workspace = true +windmill-audit.workspace = true windmill-common = { workspace = true, default-features = false } axum.workspace = true anyhow.workspace = true diff --git a/backend/windmill-api-settings/src/datatable_roles_oss.rs b/backend/windmill-api-settings/src/datatable_roles_oss.rs new file mode 100644 index 0000000000..885d13ee9d --- /dev/null +++ b/backend/windmill-api-settings/src/datatable_roles_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the data table role catalog endpoints come from: the enterprise implementation, or a +//! refusal. Roles are an Enterprise Edition feature; see `windmill_common::datatable_roles_oss`. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_roles_ee::{ + create_datatable_role, delete_datatable_role, list_datatable_roles, update_datatable_role, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +// The routes stay registered so the API has one shape; each answers after authentication, before +// anything is read. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use windmill_api_auth::ApiAuthed; + use windmill_common::{ + datatable_roles_oss::datatable_roles_unavailable as unavailable, error::Result, + }; + + pub(crate) async fn list_datatable_roles(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn create_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn update_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn delete_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } +} diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index cead2f4b27..e8d8a7124c 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -17,6 +17,9 @@ mod audit_logs_s3; mod audit_logs_s3_backfill; #[cfg(feature = "parquet")] mod background_task; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod datatable_roles_ee; +mod datatable_roles_oss; #[cfg(feature = "private")] mod ee; pub mod ee_oss; @@ -58,9 +61,11 @@ use windmill_common::{ AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, - HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, - RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, + GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, + MAX_TOKEN_EXPIRATION_DAYS_SETTING, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, + RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING, UNIQUE_ID_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WS_BASE_URL_SETTING, @@ -113,6 +118,9 @@ async fn get_ruff_config_unauthed(Extension(db): Extension) -> error::Result pub fn global_service() -> Router { #[warn(unused_mut)] let r = Router::new() + // `/local` is the path in openapi.yaml, so every generated client (getLocal) calls it; + // `/envs` stays for callers that found the route in the code. + .route("/local", get(get_local_settings)) .route("/envs", get(get_local_settings)) .route( "/global/{key}", @@ -147,6 +155,16 @@ pub fn global_service() -> Router { "/list_custom_instance_pg_databases", post(list_custom_instance_pg_databases), ) + .route( + "/datatable_roles", + get(datatable_roles_oss::list_datatable_roles) + .post(datatable_roles_oss::create_datatable_role), + ) + .route( + "/datatable_roles/{id}", + post(datatable_roles_oss::update_datatable_role) + .delete(datatable_roles_oss::delete_datatable_role), + ) .route( "/refresh_custom_instance_user_pwd", post(refresh_custom_instance_user_pwd), @@ -284,15 +302,28 @@ pub async fn test_s3_bucket( use bytes::Bytes; use futures::StreamExt; - // The probe executes on the API server itself. On multi-tenant Cloud that is a shared control - // plane, so we constrain untrusted callers to remove the SSRF / credential-exfiltration / - // local-filesystem surface (see validate_object_storage_test). On self-hosted instances the - // object store usually lives on the local/private network and all authenticated users are - // trusted, so testing there stays unrestricted. Super admins keep the unrestricted path too. + // The probe executes on the API server itself and reflects the upstream response into the + // error, so any authenticated caller could otherwise use it as an SSRF / port-scan primitive + // against the server's network, exfiltrate its ambient credentials, or write to its local + // disk (see validate_object_storage_test). That holds on self-hosted instances as much as on + // Cloud, so only super admins get the unrestricted path. let is_super_admin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; - let restrict = !is_super_admin && *CLOUD_HOSTED; + let restrict = !is_super_admin; if restrict { - validate_object_storage_test(&test_s3_bucket).await?; + validate_object_storage_test(&test_s3_bucket) + .await + .map_err(|e| match e { + // A job token never counts as a super admin (it is capped at workspace admin), so + // a super admin calling this route from a script is told why rather than that + // they lack a privilege they hold. + error::Error::NotAuthorized(msg) if authed.job_id.is_some() => { + error::Error::NotAuthorized(format!( + "{msg} A job token ($WM_TOKEN) is never treated as a super admin; call \ + this route with a user token instead." + )) + } + e => e, + })?; } let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) @@ -355,8 +386,8 @@ pub async fn test_s3_bucket( } } -// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller on -// Cloud. The probe runs on the shared API server, so without these constraints an authenticated +// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller. +// The probe runs on the API server, so without these constraints an authenticated // user could coerce the server into connecting to arbitrary internal endpoints (SSRF), signing // requests with the instance role (credential exfiltration), or reading/writing the server's local // disk (filesystem object store). @@ -366,6 +397,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul opt.as_ref().is_some_and(|s| !s.is_empty()) } + // Every refusal names the way out: the resource usually works in jobs (workers reach the + // endpoint directly), so without it the refusal reads as a broken resource. + const ALTERNATIVE: &str = + "Ask a super admin to run it, or test the resource from a script, which runs on a worker."; + // Reject backends that rely on the server's identity or local filesystem, require explicit // credentials for the rest (so the server never falls back to its own ambient credentials), and // resolve the host the client will actually connect to. We derive the *effective* endpoint here @@ -376,20 +412,25 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul let effective_endpoint: Option = match settings { ObjectSettings::Filesystem(_) => { return Err(error::Error::NotAuthorized( - "Testing a local filesystem object store requires a super admin".to_string(), + "Testing a local filesystem object store requires a super admin: it runs on the \ + Windmill server and reads and writes the server's local disk. Ask a super admin \ + to run it." + .to_string(), )); } ObjectSettings::AwsOidc(_) => { - return Err(error::Error::NotAuthorized( - "Testing OIDC-based object storage requires a super admin".to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing OIDC-based object storage requires a super admin: it runs on the \ + Windmill server with the server's own identity. {ALTERNATIVE}" + ))); } ObjectSettings::S3(s3) => { if !(non_empty(&s3.access_key) && non_empty(&s3.secret_key)) { - return Err(error::Error::NotAuthorized( - "Testing S3 storage without explicit credentials requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing S3 storage without an explicit access key and secret key requires a \ + super admin: it runs on the Windmill server, which would use its own ambient \ + credentials. {ALTERNATIVE}" + ))); } let region = s3 .region @@ -413,10 +454,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul } ObjectSettings::Azure(azure) => { if !non_empty(&azure.access_key) { - return Err(error::Error::NotAuthorized( - "Testing Azure storage without an explicit access key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing Azure storage without an explicit access key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } Some( azure @@ -432,10 +474,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul // otherwise an untrusted caller could probe with the server's identity (the very // SSRF/credential-exfil this function guards against). if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { - return Err(error::Error::NotAuthorized( - "Testing GCS storage without a service account key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing GCS storage without a service account key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } // The service-account-key JSON can override the data-plane URL (`gcs_base_url`) and the // OAuth token endpoint (`token_uri`); the GCS client connects to whatever they point at. @@ -492,10 +535,15 @@ async fn validate_public_endpoint(endpoint: &str) -> error::Result<()> { // attempts (a name resolving to both a public and a private address). for addr in addrs { if is_forbidden_ip(addr.ip()) { - return Err(error::Error::NotAuthorized( - "Testing object storage at a private, loopback, or link-local endpoint requires a super admin" - .to_string(), - )); + // The resolved address stays out of the message: it is the server's resolver's + // answer, and this message is only ever shown to the caller being constrained. + return Err(error::Error::NotAuthorized(format!( + "Testing object storage at '{host}', which resolves to a private, loopback, or \ + link-local address, requires a super admin: this test runs on the Windmill \ + server, which is not allowed to probe internal addresses for non-super-admins. \ + Ask a super admin to run it, or test the resource from a script, which runs on \ + a worker." + ))); } } Ok(()) @@ -897,6 +945,8 @@ async fn run_setting_pre_write_hook( AI_CONFIG_SETTING => { windmill_ai::ai_types::validate_model_pricing_json(value) .map_err(error::Error::BadRequest)?; + windmill_ai::ai_types::validate_token_maps_json(value) + .map_err(error::Error::BadRequest)?; } AUTOMATE_USERNAME_CREATION_SETTING => { if value.as_bool().unwrap_or(false) { @@ -1014,6 +1064,12 @@ async fn run_setting_pre_write_hook( } } } + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING => { + // Rejected at write time rather than at boot: a mistyped origin + // matches no request, so it would silently block the very app it + // names with nothing but a log line to go on. + windmill_common::global_settings::parse_allowed_origins_setting(Some(value))?; + } HTTP_ROUTE_WORKSPACED_ROUTE_SETTING => { let serde_json::Value::Bool(workspaced_route) = value else { return Err(error::Error::BadRequest(format!( @@ -1142,6 +1198,24 @@ async fn run_setting_pre_write_hook( } } } + MAX_TOKEN_EXPIRATION_DAYS_SETTING => { + windmill_common::global_settings::parse_max_token_expiration_days(Some(value)) + .map_err(|e| { + error::Error::BadRequest(format!("{MAX_TOKEN_EXPIRATION_DAYS_SETTING}: {e}")) + })?; + } + INSTANCE_BANNER_SETTING => { + match value { + // Clearing (delete row) is handled by the caller; allow it through. + serde_json::Value::Null => {} + serde_json::Value::String(s) if s.trim().is_empty() => {} + v => { + windmill_common::global_settings::validate_instance_banner(v).map_err(|e| { + error::Error::BadRequest(format!("{INSTANCE_BANNER_SETTING}: {e}")) + })?; + } + } + } _ => {} } Ok(()) @@ -1276,12 +1350,27 @@ pub async fn get_global_setting( && key != AUTOMATE_USERNAME_CREATION_SETTING && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING + // `wmill hub pull` reads it from a job, and no job token clears the gate. It binds an + // offline license only together with `license_key`, which stays gated. + && key != UNIQUE_ID_SETTING && key != HUB_ACCESSIBLE_URL_SETTING && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING && key != HTTP_ROUTE_WORKSPACED_ROUTE_SETTING + // The route editor shows the inherited default to whoever is editing a + // trigger, who is usually not a superadmin. Not a secret either: any + // browser discovers the list by reading Access-Control-Allow-Origin off + // a response. + && key != HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING && key != WS_BASE_URL_SETTING + && key != INSTANCE_BANNER_SETTING + // The token form reads it to stop offering expirations the server would shorten. + && key != MAX_TOKEN_EXPIRATION_DAYS_SETTING + // Whoever is wiring up an MCP client reads it to know whether a URL-borne token + // would be refused, and they are usually not a superadmin. Not a secret: pointing + // any MCP client at the instance discovers the same answer. + && key != MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING { require_super_admin(&db, &authed).await?; } @@ -1593,6 +1682,8 @@ struct CustomInstanceDbLogs { replication_user: String, #[serde(skip_serializing_if = "Option::is_none")] replication_user_error: Option, + #[serde(skip_serializing_if = "String::is_empty")] + user_connect: String, } async fn list_custom_instance_pg_databases( @@ -1810,9 +1901,57 @@ async fn setup_custom_instance_pg_database_inner( } } + // Everything above logged in as the DATABASE_URL user, but whatever uses the database logs + // in as custom_instance_user. A proxy that routes on the login name can accept one and + // refuse the other, which would otherwise surface only once a data table first connects. + let user_creds = PgDatabase { + user: Some(windmill_common::datatable_roles::CUSTOM_INSTANCE_USER.to_string()), + password: Some(windmill_common::utils::get_custom_pg_instance_password(db).await?), + ..pg_creds + }; + let (client, connection) = user_creds + .connect(Some(db)) + .await + .map_err(|e| custom_instance_user_connect_error(&user_creds.host, dbname, e))?; + let join_handle = tokio::spawn(async move { connection.await }); + logs.user_connect = "OK".to_string(); + drop(client); // /!\ Drop before joining to avoid deadlock + windmill_common::shutdown_pg_connection(join_handle).await?; + Ok(()) } +fn custom_instance_user_connect_error(host: &str, dbname: &str, e: error::Error) -> error::Error { + let cause = match &e { + error::Error::Anyhow { error, .. } => format!("{error:#}"), + e => e.to_string(), + }; + // Supavisor, Supabase's pooler, reads the tenant to route to from the login + // (`.`), and only knows the logins configured for that tenant. + let lower = cause.to_lowercase(); + let routing_refused = [ + "enoidentifier", + "tenant identifier", + "tenant or user", + "tenant/user", + ] + .iter() + .any(|signature| lower.contains(signature)); + if routing_refused { + error::Error::BadConfig(format!( + "DATABASE_URL reaches Postgres through a connection pooler ({host}) that picks the \ + server to route to from the login name, and it refused custom_instance_user, the \ + role Windmill uses for instance databases ({cause}). Instance databases cannot be \ + used through this pooler: use your own Postgres database instead, or point \ + DATABASE_URL at the Postgres server directly rather than at the pooler." + )) + } else { + error::Error::ExecutionErr(format!( + "Could not connect to {dbname} as custom_instance_user: {cause}" + )) + } +} + async fn drop_custom_instance_pg_database( authed: ApiAuthed, Extension(db): Extension, @@ -2004,6 +2143,22 @@ struct CachedResourceType { #[allow(dead_code)] app: String, description: Option, + /// Doubly optional, and read through a wrapping deserializer: this struct also + /// decodes the on-disk cache, where an absent key means "written before the + /// column, leave the stored extension alone" and an explicit null means the hub + /// dropped it. Plain serde folds both into `None`. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + format_extension: Option>, + /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit + /// null (the hub naming nothing) clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } #[derive(serde::Deserialize)] @@ -2013,6 +2168,13 @@ struct HubResourceTypeRaw { schema: Option, app: String, description: Option, + #[serde(default)] + format_extension: Option, + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2054,6 +2216,8 @@ async fn fetch_resource_types_from_hub() -> error::Result 100 => None, + other => other.clone(), + }; let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .fetch_one(&db) .await?; @@ -2120,13 +2294,30 @@ async fn sync_cached_resource_types( } sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // Whether the payload carried the key at all is what decides: present + // (even as null) is authoritative and may clear, absent means a cache + // written before the column and must leave the stored value alone. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $6, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + -- A fileset is a set of files, so it cannot also be one file. + -- Create and update reject the pair; this writer bypasses both, so + -- it declines the extension rather than persisting the forbidden + -- combination onto a same-named local fileset. + format_extension = CASE + WHEN resource_type.is_fileset THEN NULL + WHEN $5 THEN EXCLUDED.format_extension + ELSE resource_type.format_extension END, + display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, + edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .execute(&db) .await?; @@ -2163,6 +2354,34 @@ mod tests { use std::collections::BTreeMap; use windmill_common::instance_config::{GlobalSettings, InstanceConfig, WorkerGroupConfig}; + #[test] + fn supavisor_refusing_custom_instance_user_is_named() { + use windmill_common::error::{to_anyhow, Error}; + let connect_error = |message: &str| { + let e = Error::from(to_anyhow(std::io::Error::other(message.to_string()))); + super::custom_instance_user_connect_error( + "aws-0-eu-west-1.pooler.supabase.com", + "dt", + e, + ) + }; + for supavisor in [ + "db error: FATAL: (ENOIDENTIFIER) no tenant identifier provided", + "db error: FATAL: Tenant or user not found", + ] { + assert!( + matches!(connect_error(supavisor), Error::BadConfig(m) if m.contains("connection pooler")), + "{supavisor}" + ); + } + assert!(matches!( + connect_error( + "db error: FATAL: password authentication failed for user \"custom_instance_user\"" + ), + Error::ExecutionErr(_) + )); + } + #[test] fn instance_config_yaml_round_trip() { let config = InstanceConfig { diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs index ecbc38fcb7..82080cb61c 100644 --- a/backend/windmill-api-settings/src/log_cleanup.rs +++ b/backend/windmill-api-settings/src/log_cleanup.rs @@ -32,7 +32,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE}; use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES_LOADED, }; use windmill_object_store::object_store_reexports::{ @@ -249,7 +249,7 @@ async fn cleanup_service_logs( // Count candidates upfront for progress reporting. let total: i64 = sqlx::query_scalar!( "SELECT COUNT(*) FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), ) .fetch_one(db) .await? @@ -274,7 +274,7 @@ async fn cleanup_service_logs( WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval LIMIT $2 ) RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), SERVICE_LOG_BATCH, ) .fetch_all(db) @@ -680,9 +680,10 @@ async fn cleanup_s3_orphans( ) -> error::Result<()> { let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed); let now = Utc::now(); - // Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS), - // so we scan for service-log orphans regardless of JOB_RETENTION_SECS. - let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + // Service logs always have a retention, so we scan for service-log orphans regardless of + // JOB_RETENTION_SECS. + let service_cutoff = + now - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); // Job-log orphans are only considered once past a job's effective retention window. That window // is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8ea3bd0995..33768f0e6b 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -19,7 +19,7 @@ use windmill_api_auth::ApiAuthed; pub use windmill_api_auth::Tokened; -use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use argon2::{Argon2, PasswordVerifier}; use axum::{ extract::{Extension, Path, Query}, response::{IntoResponse, Response}, @@ -46,15 +46,18 @@ use tracing::Instrument; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; -use windmill_common::auth::{safe_token_prefix, TOKEN_PREFIX_LEN}; -use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; +use windmill_common::auth::{hash_token, safe_token_prefix, TOKEN_PREFIX_LEN}; +use windmill_common::global_settings::{ + load_value_from_global_settings, parse_max_token_expiration_days, + AUTOMATE_USERNAME_CREATION_SETTING, MAX_TOKEN_EXPIRATION_DAYS_SETTING, +}; use windmill_common::oauth2::InstanceEvent; use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::users::{ - username_to_permissioned_as, PERMISSIONED_AS_MAX_LEN, SUPERADMIN_NOTIFICATION_EMAIL, - SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, + username_to_permissioned_as, EMAIL_COLUMN_MAX_LEN, PERMISSIONED_AS_MAX_LEN, + SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, }; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; @@ -140,6 +143,16 @@ pub fn global_service() -> Router { ) .route("/tokens/list", get(list_tokens)) .route("/tokens/impersonate", post(impersonate)) + .route("/login_links", post(create_login_link)) + .route( + "/cloud_trial_offer", + post(set_cloud_trial_offer).get(get_cloud_trial_offer), + ) + .route("/cloud_trial_offer/go", post(go_cloud_trial_offer)) + .route( + "/onboarding_profile", + post(set_onboarding_profile).get(get_onboarding_profile), + ) .route("/usage", get(get_usage)) .route("/all_runnables", get(get_all_runnables)) .route("/refresh_token", get(refresh_token)) @@ -158,6 +171,7 @@ pub fn make_unauthed_service() -> Router { .route("/logout", post(logout).get(logout)) .route("/is_first_time_setup", get(is_first_time_setup)) .route("/request_password_reset", post(request_password_reset)) + .route("/login_link/{token}", get(consume_login_link)) .route("/is_smtp_configured", get(is_smtp_configured)) .route( "/is_password_login_disabled", @@ -255,11 +269,14 @@ pub struct WorkspaceInvite { #[derive(Deserialize)] pub struct NewUser { pub email: String, - pub password: String, + /// Required when `login_type` is `password` (the default), ignored otherwise. + pub password: Option, pub super_admin: bool, pub name: Option, pub company: Option, pub skip_email: Option, + /// `password`, `pending_oauth`, or a configured OAuth login client key. + pub login_type: Option, } #[derive(Deserialize)] @@ -510,7 +527,7 @@ async fn list_users_as_super_admin( let rows = if active_only.is_some_and(|x| x) { sqlx::query_as!( GlobalUserInfo, - r#"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), + r#"WITH active_users AS (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) SELECT email as "email!", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as "verified!", super_admin as "super_admin!", devops as "devops!", name, company, username, first_time_user as "first_time_user!", role_source as "role_source!", disabled as "disabled!", NULL::text as workspace_id FROM password @@ -657,15 +674,17 @@ async fn logout( let t_prefix = token.get(..TOKEN_PREFIX_LEN).unwrap_or(&token); let email = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT { - sqlx::query_scalar!( + // A guest's browser session is a session too: this is its one user-driven revocation. + sqlx::query_scalar::<_, Option>( "WITH email_lookup AS ( SELECT email FROM token WHERE token_hash = $1 ) DELETE FROM token - WHERE email = (SELECT email FROM email_lookup) AND label = 'session' + WHERE email = (SELECT email FROM email_lookup) + AND label IN ('session', 'guest_session') RETURNING email", - t_hash ) + .bind(&t_hash) .fetch_optional(&mut *tx) .await? } else { @@ -745,7 +764,30 @@ async fn whoami( Path(w_id): Path, authed: ApiAuthed, ) -> JsonResult { + let is_guest = windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()); let ApiAuthed { username, email, is_admin, groups, folders, .. } = authed; + // A guest would otherwise fall through to the non-member branch below and be + // handed a `superadmin` role. Answer it here, as the operator-shaped identity it is. + if is_guest { + return Ok(Json(UserInfo { + workspace_id: w_id, + email, + username, + name: None, + is_admin: false, + is_super_admin: false, + created_at: chrono::Utc::now(), + groups: vec![], + operator: true, + disabled: false, + role: Some("guest".to_string()), + folders_read: vec![], + folders: vec![], + folders_owners: vec![], + is_service_account: false, + non_member: true, + })); + } let user = get_user(&w_id, &username, &db).await?; // Only treat the row as "this user is a member" when its email matches; the // derived username is instance-unique so a match on a different email should @@ -1239,6 +1281,7 @@ async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Resu sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &authed.email).await?; audit_log( &mut *tx, @@ -1661,15 +1704,27 @@ async fn delete_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?; - let usernames = sqlx::query_scalar!( - "DELETE FROM usr WHERE email = $1 RETURNING username", + let memberships = sqlx::query!( + "DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id", &email_to_delete ) .fetch_all(&mut *tx) .await?; - for username in usernames { + for row in memberships { + let username = row.username; + // A tenant list names a principal of its workspace, so the name has to be freed in every + // workspace this account belonged to: a later account taking the username would otherwise + // inherit the data table access it had. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &row.workspace_id, + &format!("u/{username}"), + ) + .await?; + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; @@ -1717,7 +1772,6 @@ struct ChangeUserEmail { /// `varchar(50)`, and `v2_job.permissioned_as` in a `varchar(55)`; every other email column is /// `varchar(255)`. The strictest of the two bounds is used for all of them. const SHORT_EMAIL_COLUMN_MAX_LEN: usize = 50; -const EMAIL_COLUMN_MAX_LEN: usize = 255; /// Move an account to a new email address, in place: the `password` row (and with it the /// instance-wide username, the role and the login type) is kept and every email-keyed row is @@ -1869,7 +1923,7 @@ async fn change_user_email( .execute(&mut *tx) .await?; - // ---- account ---- (draft.email follows through its ON UPDATE CASCADE fkey) + // ---- account ---- sqlx::query!( "UPDATE password SET email = $1 WHERE email = $2", &new_email, @@ -1883,6 +1937,7 @@ async fn change_user_email( } _ => e.into(), })?; + windmill_common::user_drafts::rename_drafts_of_email(&mut *tx, &old_email, &new_email).await?; sqlx::query!( "UPDATE usr SET email = $1 WHERE email = $2", @@ -2121,6 +2176,28 @@ async fn change_user_email( .execute(&mut *tx) .await?; + // An app draft carries a copy of the deployed policy, principal included. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2"#, + &new_principal, + &old_principal + ) + .execute(&mut *tx) + .await?; + + // A raw-app draft persists the address the client read back too. The deploy sends it beside + // the principal, where an address naming somebody else is rejected — and unlike a live read + // it never refreshes on its own. Same group guard as the deployed policy above, plus the + // `IS NULL` arm: without it the predicate is `NULL` for a draft with no principal, which is + // neither true nor false, so those rows would be skipped. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($1::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of_email' = $2 AND (value->'policy'->>'on_behalf_of' IS NULL OR value->'policy'->>'on_behalf_of' NOT LIKE 'g/%')"#, + &new_email, + &old_email + ) + .execute(&mut *tx) + .await?; + // A folder's default rules are an ordered array, first match wins, so the rewrite has to // preserve their order. A rule left on the old address makes `ensure_permissioned_as_exists` // reject the creation of every runnable the rule matches. @@ -2298,9 +2375,9 @@ async fn change_user_email( ) .await?; - // Read back inside the transaction: the address is derived at dispatch through a cache - // that nothing else evicts, so without this a job pushed in the next 60s would resolve - // the old address and with it the wrong superadmin flag and instance groups. + // Read back inside the transaction so this process can evict its own keys immediately. + // `notify_user_email_change` reaches every replica for the same change, but asynchronously, + // and this one is the replica that just served the request. let memberships = sqlx::query_scalar!("SELECT workspace_id FROM usr WHERE email = $1", &new_email) .fetch_all(&mut *tx) @@ -2393,6 +2470,15 @@ pub async fn delete_workspace_user_internal( tx: &mut Transaction<'_, Postgres>, authed: Option<&ApiAuthed>, // None for system operations ) -> Result<()> { + // Same reasoning as the `extra_perms` sweep below: a freed username must not stay named + // anywhere that grants access, tenant lists included. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + tx, + w_id, + &format!("u/{username_to_delete}"), + ) + .await?; + // ---- Clean up extra_perms referencing this user ---- let extra_perms_tables = [ "script", @@ -2680,10 +2766,8 @@ async fn login( .await?; if let Some((email, hash, super_admin)) = email_w_h { - let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; if argon2 - .verify_password(password.as_bytes(), &parsed_hash) + .verify_password(password.as_bytes(), hash.as_str()) .is_err() { audit_log( @@ -2881,7 +2965,12 @@ pub async fn create_session_token<'c>( .execute(&mut **tx) .await?; - let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); + set_session_cookie(&cookies, &token, *MAX_SESSION_VALIDITY_SECONDS); + Ok(token) +} + +fn set_session_cookie(cookies: &Cookies, token: &str, validity_seconds: i64) { + let mut cookie = Cookie::new(COOKIE_NAME, token.to_string()); cookie.set_secure(IS_SECURE.load(std::sync::atomic::Ordering::Relaxed)); cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); cookie.set_http_only(true); @@ -2891,19 +2980,183 @@ pub async fn create_session_token<'c>( } let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); - expire += time::Duration::seconds(*MAX_SESSION_VALIDITY_SECONDS); + expire += time::Duration::seconds(validity_seconds); cookie.set_expires(expire); cookies.add(cookie); +} + +lazy_static::lazy_static! { + /// A guest session is the only credential held by someone with no account, so + /// there is nothing to disable when the workspace revokes guest access or the + /// identity provider removes them — the expiry is the revocation. Much shorter + /// than a member session for that reason. + static ref GUEST_SESSION_VALIDITY_SECONDS: i64 = std::env::var("GUEST_SESSION_VALIDITY_SECONDS") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(8 * 60 * 60); +} + +/// Mint a browser session for someone the identity provider authenticated who is a +/// member of no workspace, so they can open one guest-mode app. Writes no `password` +/// and no `usr` row: that absence is what keeps a guest off every seat counter, so +/// nothing here may be "helpfully" upgraded into provisioning. +/// +/// Pinned to `w_id` (`AuthCache` matches on `token.workspace_id`): without the pin an +/// `apps:run:` scope would unlock a same-path app elsewhere. So a guest cannot +/// authenticate on any workspace-less route (`/api/users/*`, `/api/settings/*`); a +/// page that needs one for a guest must become workspace-scoped, not loosen the pin. +/// +/// Refuses unless every gate says yes (`guest_app_admits`, then the allowance in +/// `guest_admission`), so no caller can mint where a guest is not wanted, whatever it +/// believed when it decided to call. All that is left to the caller is the +/// authentication of `email`. +pub async fn create_guest_session_token<'c>( + email: &str, + w_id: &str, + app_path: &str, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + cookies: Cookies, +) -> Result { + use windmill_common::min_version::MIN_VERSION_SUPPORTS_TOKEN_HASH; + + let token = rd_string(32); + let t_hash = windmill_common::auth::hash_token(&token); + let t_prefix = token.get(..TOKEN_PREFIX_LEN).unwrap_or(&token); + let plaintext: Option<&str> = if MIN_VERSION_SUPPORTS_TOKEN_HASH.met().await { + None + } else { + Some(&token) + }; + let scopes = windmill_api_auth::scopes::guest_session_scopes(app_path)?; + + // No account at all (see `has_any_account`): an account holder is refused a guest + // session, never handed a second, cheaper identity. The same helper the JWT arm uses. + if windmill_common::users::has_any_account(&mut **tx, email).await? { + return Err(Error::NotAuthorized( + "an existing account cannot hold a guest session".to_string(), + )); + } + if !windmill_common::workspaces::guest_app_admits(&mut **tx, w_id, app_path).await? { + return Err(Error::NotAuthorized(format!( + "app {app_path} is not open to guests" + ))); + } + windmill_common::workspaces::guest_admission(&mut **tx, email).await?; + + sqlx::query!( + "INSERT INTO token + (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, now() + ($6 || ' seconds')::interval, false, $7, $8)", + t_hash, + t_prefix, + plaintext as Option<&str>, + email, + windmill_common::auth::GUEST_SESSION_LABEL, + &GUEST_SESSION_VALIDITY_SECONDS.to_string(), + &scopes, + w_id, + ) + .execute(&mut **tx) + .await?; + + // The only durable record that a guest was here, and the set the allowance is + // counted on; not the audit log, see the migration. Idempotent per email, + // workspace and day. + sqlx::query!( + "INSERT INTO guest_activity (email, workspace_id, day) + VALUES ($1, $2, CURRENT_DATE) + ON CONFLICT (email, workspace_id, day) + DO UPDATE SET last_seen_at = now()", + email, + w_id, + ) + .execute(&mut **tx) + .await?; + + audit_log( + &mut **tx, + &AuditAuthor { + email: email.to_string(), + username: email.to_string(), + username_override: None, + token_prefix: Some(t_prefix.to_string()), + }, + "users.login_guest", + ActionKind::Create, + w_id, + Some(app_path), + Some([("entry", "idp")].into()), + ) + .await?; + + set_session_cookie(&cookies, &token, *GUEST_SESSION_VALIDITY_SECONDS); Ok(token) } // create_token_internal is re-exported from windmill-api-auth above +/// Applies the instance-wide ceiling on how long a token a caller picks the lifetime of may +/// live (`create_token`, and `impersonate` for superadmins), returning the expiration to store: +/// the requested one while it fits, the ceiling otherwise, and the ceiling as well when none was +/// requested. Only the stored expiration is capped: tokens already stored when the setting is +/// turned on or lowered keep theirs, since the auth lookup never reads the setting. +/// +/// It shortens rather than refuses because most callers do not comply on their own. The CLI +/// authorization page, `wmill user create-token` and the editor's language-server token each +/// pick a lifetime, often none at all, without reading the setting (and CLIs already installed +/// never will), so refusing would break logging in and the editor instead of the long-lived +/// tokens the setting is aimed at. +/// +/// Read from `global_settings` on each call rather than cached: token creation is rare +/// enough that the round trip costs nothing, and the ceiling is then never served stale. +/// +/// A token owned by a service account is exempt: in the workspace the token names, or in any +/// workspace for a workspace-less token, which has none to match. Service accounts are the +/// identity automation that needs a long-lived credential runs as. The cost is that any +/// workspace admin can create and impersonate one to hold an uncapped token, so the ceiling +/// bounds personal tokens rather than what an admin can obtain. +async fn cap_token_expiration( + db: &DB, + owner_email: &str, + workspace_id: Option<&str>, + requested: Option>, +) -> Result>> { + let value = load_value_from_global_settings(db, MAX_TOKEN_EXPIRATION_DAYS_SETTING).await?; + let max_days = match parse_max_token_expiration_days(value.as_ref()) { + Ok(Some(max_days)) => max_days, + Ok(None) => return Ok(requested), + // Both write paths reject this, so only a row written around them gets here. + Err(e) => { + tracing::warn!("ignoring {MAX_TOKEN_EXPIRATION_DAYS_SETTING}: {e}"); + return Ok(requested); + } + }; + let max = chrono::Utc::now() + chrono::Duration::days(max_days); + + let is_service_account = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE email = $1 AND is_service_account IS true + AND ($2::varchar IS NULL OR workspace_id = $2))", + owner_email, + workspace_id, + ) + .fetch_one(db) + .await? + .unwrap_or(false); + if is_service_account { + return Ok(requested); + } + + Ok(Some(match requested { + Some(expiration) if expiration < max => expiration, + _ => max, + })) +} + async fn create_token( Extension(db): Extension, authed: ApiAuthed, OptJobAuthed { job_id, .. }: OptJobAuthed, - Json(token_config): Json, + Json(mut token_config): Json, ) -> Result<(StatusCode, String)> { forbid_elevated_job_token(&db, &authed.email, job_id).await?; check_token_create_rate_limit(&authed.username)?; @@ -2925,6 +3178,14 @@ async fn create_token( windmill_api_auth::ensure_scopes_within_caller(&authed, token_config.scopes.as_deref())?; + token_config.expiration = cap_token_expiration( + &db, + &authed.email, + token_config.workspace_id.as_deref(), + token_config.expiration, + ) + .await?; + let mut tx = db.begin().await?; let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?; @@ -2981,6 +3242,7 @@ async fn impersonate( .fetch_optional(&db) .await? .unwrap_or(false); + let expiration = cap_token_expiration(&db, &impersonated, None, new_token.expiration).await?; let mut tx = db.begin().await?; sqlx::query!( @@ -2992,7 +3254,7 @@ async fn impersonate( plaintext as Option<&str>, impersonated, new_token.label, - new_token.expiration, + expiration, is_super_admin ) .execute(&mut *tx) @@ -3002,7 +3264,7 @@ async fn impersonate( &mut *tx, &t_hash, new_token.label.as_deref(), - new_token.expiration, + expiration, ) .await; @@ -3021,6 +3283,506 @@ async fn impersonate( Ok((StatusCode::CREATED, token)) } +const LOGIN_LINK_DEFAULT_TTL_S: u32 = 600; +const LOGIN_LINK_MAX_TTL_S: u32 = 900; +const LOGIN_LINK_DEFAULT_RD: &str = "/user/workspaces"; +const LOGIN_LINK_EXPIRED_PAGE: &str = "/user/login_link_expired"; + +#[derive(Deserialize)] +pub struct NewLoginLink { + pub email: String, + pub expires_in_s: Option, + pub rd: Option, + /// Refuse to mint unless the account still has this login type: a caller re-entering an + /// account it created can require `pending_oauth`, so the link stops working once the + /// owner has set a password or signed in with a provider. + pub require_login_type: Option, +} + +#[derive(Serialize)] +pub struct LoginLink { + pub url: String, + pub expires_at: chrono::DateTime, +} + +/// A post-login destination is only ever a same-origin path: anything else would hand the +/// fresh session's first navigation to another host. Control characters are refused because +/// browsers strip tab/newline from a `Location` before parsing it, so `/\t/host` reads as +/// the protocol-relative `//host`. +fn same_origin_rd(rd: Option) -> Option { + rd.filter(|r| { + r.starts_with('/') + && !r.starts_with("//") + && !r.contains('\\') + && !r.chars().any(|c| c.is_ascii_control()) + }) +} + +#[cfg(test)] +mod same_origin_rd_tests { + use super::same_origin_rd; + + fn accepts(rd: &str) -> bool { + same_origin_rd(Some(rd.to_string())).is_some() + } + + #[test] + fn only_plain_same_origin_paths_pass() { + assert!(accepts("/")); + assert!(accepts("/user/workspaces?rd=%2Fx")); + assert!(!accepts("https://evil.example/")); + assert!(!accepts("//evil.example/")); + assert!(!accepts("/\\evil.example/")); + assert!(!accepts("/\t/evil.example/")); + assert!(!accepts("/x\r\nSet-Cookie: a=b")); + assert!(!accepts("user/workspaces")); + } +} + +/// Both provisioning writes reference `password(email)`; a typo'd address from the +/// provisioning script should read as "no such account", not as a foreign-key error. +async fn require_account( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + email: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if !exists { + return Err(Error::NotFound(format!("no account for {email}"))); + } + Ok(()) +} + +fn login_link_redirect(location: String) -> Response { + ( + StatusCode::FOUND, + [ + ("location", location), + ("referrer-policy", "no-referrer".to_string()), + ], + ) + .into_response() +} + +/// Mint a single-use link that signs `email` in when opened. The row is not a `token`: +/// it can only ever become a session, and burning it needs no cache invalidation. +async fn create_login_link( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(nl): Json, +) -> Result<(StatusCode, Json)> { + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + + let email = nl.email.to_lowercase(); + let rd = match nl.rd { + Some(rd) => Some(same_origin_rd(Some(rd)).ok_or_else(|| { + Error::BadRequest("rd must be a same-origin path starting with /".to_string()) + })?), + None => None, + }; + let ttl = nl + .expires_in_s + .unwrap_or(LOGIN_LINK_DEFAULT_TTL_S) + .clamp(1, LOGIN_LINK_MAX_TTL_S); + + let mut tx = db.begin().await?; + let target = sqlx::query!( + "SELECT super_admin, devops, login_type FROM password WHERE email = $1 AND disabled = false", + &email + ) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + return Err(Error::NotFound(format!("no active account for {email}"))); + }; + // A link is a full session for its account; whoever holds the minting credential + // must not be able to turn it into an instance-wide role. + if target.super_admin || target.devops { + return Err(Error::BadRequest( + "login links cannot target superadmin or devops accounts".to_string(), + )); + } + if let Some(required) = nl.require_login_type.as_deref() { + if target.login_type != required { + return Err(Error::Generic( + StatusCode::CONFLICT, + format!( + "login_type_mismatch: {email} signs in with {}, not {required}", + target.login_type + ), + )); + } + } + + let token = rd_string(32); + let expires_at = chrono::Utc::now() + chrono::Duration::seconds(ttl as i64); + sqlx::query!( + "INSERT INTO login_link (token_hash, email, rd, expiration, created_by) + VALUES ($1, $2, $3, $4, $5)", + hash_token(&token), + &email, + rd, + expires_at, + &authed.email, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.login_link.create", + ActionKind::Create, + "global", + Some(&email), + Some([("expires_in_s", &ttl.to_string()[..])].into()), + ) + .await?; + tx.commit().await?; + + let url = format!( + "{}/api/auth/login_link/{}", + (**BASE_URL.load()).clone(), + token + ); + Ok((StatusCode::CREATED, Json(LoginLink { url, expires_at }))) +} + +#[derive(Deserialize)] +pub struct CloudTrialOfferUpdate { + pub email: String, + #[serde(default)] + pub consumed: bool, +} + +#[derive(Deserialize)] +pub struct OnboardingProfileUpdate { + pub email: String, + pub profile: serde_json::Value, +} + +#[derive(Serialize)] +pub struct OnboardingProfile { + pub profile: Option, +} + +/// Context the invite carried about this account's owner, written at provisioning. +/// Onboarding tailors itself from it (today: `touch_point` answers the source question +/// so it is never asked); everything degrades to the plain flow when absent. +async fn set_onboarding_profile( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(body): Json, +) -> Result { + if !*CLOUD_HOSTED { + return Err(Error::NotFound("cloud only".to_string())); + } + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + if !body.profile.is_object() { + return Err(Error::BadRequest( + "profile must be a JSON object".to_string(), + )); + } + let email = body.email.to_lowercase(); + let mut tx = db.begin().await?; + require_account(&mut tx, &email).await?; + sqlx::query!( + "INSERT INTO cloud_onboarding_profile (email, profile, created_by) VALUES ($1, $2, $3) + ON CONFLICT (email) DO UPDATE SET profile = EXCLUDED.profile", + &email, + body.profile, + &authed.email + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "users.onboarding_profile.set", + ActionKind::Update, + "global", + Some(&email), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("onboarding profile for {email} recorded")) +} + +async fn get_onboarding_profile( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + if !*CLOUD_HOSTED { + return Ok(Json(OnboardingProfile { profile: None })); + } + let profile = sqlx::query_scalar!( + "SELECT profile FROM cloud_onboarding_profile WHERE email = $1", + &authed.email + ) + .fetch_optional(&db) + .await?; + Ok(Json(OnboardingProfile { profile })) +} + +#[derive(Serialize)] +pub struct CloudTrialOffer { + pub offered: bool, +} + +/// What the customer portal answers when asked to sign a cloud account in and start its +/// pre-approved trial. +pub enum PortalTrialLogin { + /// Send the browser here: a short-lived portal login that starts the trial on landing. + LoginUrl(String), + /// The portal will not start one (a subscription exists, or it knows no offer); the + /// offer is spent and the browser goes to the portal home instead. + Unavailable { reason: String, portal_url: String }, +} + +async fn set_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(body): Json, +) -> Result { + if !*CLOUD_HOSTED { + return Err(Error::NotFound("cloud only".to_string())); + } + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + let email = body.email.to_lowercase(); + let mut tx = db.begin().await?; + require_account(&mut tx, &email).await?; + if body.consumed { + sqlx::query!( + "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + &email + ) + .execute(&mut *tx) + .await?; + } else { + // A consumed offer stays consumed: a trial or subscription already exists for it. + sqlx::query!( + "INSERT INTO cloud_trial_offer (email, created_by) VALUES ($1, $2) + ON CONFLICT (email) DO NOTHING", + &email, + &authed.email + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.cloud_trial_offer.set", + ActionKind::Update, + "global", + Some(&email), + Some([("consumed", if body.consumed { "true" } else { "false" })].into()), + ) + .await?; + tx.commit().await?; + Ok(format!( + "cloud trial offer for {email} {}", + if body.consumed { + "consumed" + } else { + "recorded" + } + )) +} + +async fn offered(db: &DB, email: &str) -> Result { + Ok(sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM cloud_trial_offer WHERE email = $1 AND consumed_at IS NULL)", + email + ) + .fetch_one(db) + .await? + .unwrap_or(false)) +} + +async fn get_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + if !*CLOUD_HOSTED { + return Ok(Json(CloudTrialOffer { offered: false })); + } + Ok(Json(CloudTrialOffer { + offered: offered(&db, &authed.email).await?, + })) +} + +/// Where the browser goes to start the pre-approved trial: a signed-in portal login, or +/// the portal's front page with the reason it could not start one. +#[derive(Serialize)] +pub struct CloudTrialOfferGo { + pub location: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// The one click that turns a cloud account's pre-approved offer into a trial: the portal +/// is asked for a login that starts it, and the browser is handed over. The portal is the +/// authority on whether the offer still stands; its refusal spends the offer here so the +/// sidebar stops advertising it. +async fn go_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + // The answer carries a signed-in portal login for this account: a credential for + // another system, and asking for it starts the trial. A script running as the offered + // user holds their identity through `$WM_TOKEN`, so a job token must not be able to + // fetch it and hand it to whoever wrote the script. It is a POST answered as JSON, not + // a redirecting GET, so a cross-site top-level navigation cannot start the trial with + // the SameSite=Lax session cookie either; the frontend navigates to `location` itself. + if authed.job_id.is_some() { + return Err(Error::NotAuthorized( + "This endpoint cannot be called with a job token ($WM_TOKEN).".to_string(), + )); + } + if !*CLOUD_HOSTED || !offered(&db, &authed.email).await? { + return Err(Error::NotFound( + "no pre-approved trial offer for this account".to_string(), + )); + } + let outcome = crate::users_oss::portal_cloud_trial_login(&authed.email).await?; + let (location, reason) = match outcome { + PortalTrialLogin::LoginUrl(url) => (url, None), + PortalTrialLogin::Unavailable { reason, portal_url } => (portal_url, Some(reason)), + }; + let mut tx = db.begin().await?; + if reason.is_some() { + sqlx::query!( + "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + &authed.email + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.cloud_trial_offer.go", + ActionKind::Execute, + "global", + Some(&authed.email), + Some([("outcome", reason.as_deref().unwrap_or("login"))].into()), + ) + .await?; + tx.commit().await?; + Ok(Json(CloudTrialOfferGo { location, reason })) +} + +#[derive(Deserialize)] +struct LoginLinkQuery { + rd: Option, +} + +async fn consume_login_link( + headers: axum::http::HeaderMap, + cookies: Cookies, + Extension(db): Extension, + Path(token): Path, + Query(query): Query, +) -> Result { + let bounce = |reason: &str| { + Ok(login_link_redirect(format!( + "{LOGIN_LINK_EXPIRED_PAGE}?reason={reason}" + ))) + }; + if token.len() != 32 { + return bounce("invalid"); + } + let t_hash = hash_token(&token); + // The account is unknown until the row is read, so only the global and per-IP tiers + // apply here; a 32-char random token leaves nothing for the per-account tier to guard. + windmill_common::login_rate_limit::check_and_increment_login_attempt( + &headers, + &t_hash[..TOKEN_PREFIX_LEN], + )?; + + let mut tx = db.begin().await?; + let link = sqlx::query!( + "UPDATE login_link SET consumed_at = now() + WHERE token_hash = $1 AND consumed_at IS NULL AND expiration > now() + RETURNING email, rd, created_by", + &t_hash + ) + .fetch_optional(&mut *tx) + .await?; + let Some(link) = link else { + let used = sqlx::query_scalar!( + "SELECT consumed_at IS NOT NULL AS \"used!\" FROM login_link WHERE token_hash = $1", + &t_hash + ) + .fetch_optional(&mut *tx) + .await?; + return bounce(match used { + Some(true) => "used", + Some(false) => "expired", + None => "invalid", + }); + }; + + // Re-checked at open time and locked through session creation: a promotion inside + // the link's window must not turn a link minted for an ordinary account into a + // privileged session. The bounce drops the transaction, so the link is not spent. + let target = sqlx::query!( + "SELECT super_admin, devops FROM password WHERE email = $1 AND disabled = false FOR UPDATE", + &link.email + ) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + return bounce("invalid"); + }; + if target.super_admin || target.devops { + return bounce("invalid"); + } + + let session = create_session_token(&link.email, false, None, false, &mut tx, cookies).await?; + audit_log( + &mut *tx, + &AuditAuthor { + email: link.email.clone(), + username: link.email.clone(), + username_override: None, + token_prefix: Some(safe_token_prefix(&session)), + }, + "users.login", + ActionKind::Create, + "global", + Some(&truncate_token(&session)), + Some( + [ + ("method", "login_link"), + ("minted_by", link.created_by.as_str()), + ] + .into(), + ), + ) + .await?; + tx.commit().await?; + + let rd = link + .rd + .or_else(|| same_origin_rd(query.rd)) + .unwrap_or_else(|| LOGIN_LINK_DEFAULT_RD.to_string()); + Ok(login_link_redirect(rd)) +} + #[derive(Deserialize)] pub struct ImpersonateServiceAccountRequest { pub username: String, @@ -3158,9 +3920,13 @@ async fn update_token_scopes( let mut tx = db.begin().await?; + // A guest-labelled token is never rescoped: its scopes are its whole confinement, + // and after promotion the same email owns an account that could otherwise strip + // them from the still-valid guest credential. Same shape as the relabel guard. let updated: Option = sqlx::query_scalar!( "UPDATE token SET scopes = $1 WHERE email = $2 AND token_prefix = $3 + AND (label IS NULL OR label <> 'guest_session') RETURNING token_prefix", req.scopes.as_deref(), &authed.email, @@ -3171,7 +3937,7 @@ async fn update_token_scopes( let prefix = updated.ok_or_else(|| { Error::NotFound(format!( - "token {token_prefix} not found or not owned by user" + "token {token_prefix} not found, not owned by user, or not rescopable" )) })?; @@ -3205,8 +3971,8 @@ async fn update_token_label( Path(token_prefix): Path, Json(req): Json, ) -> Result { - // The new label must not collide with a system-token namespace (`session`, - // `ephemeral*`, `debugger-token`, `mcp-oauth-*`): those labels are + // The new label must not collide with a system-token namespace (see + // `windmill_common::auth::is_user_token`): those labels are // load-bearing, and a user-set collision would orphan the token — hidden // from the UI (`isUserToken`) and rejected by the editability guard below — // while it still authenticates. (`is_user_token(None)` is true, so clearing @@ -3241,9 +4007,14 @@ async fn update_token_label( WHERE email = $2 AND token_prefix = $3 AND (label IS NULL OR ( label <> 'session' + AND label <> 'guest_session' AND lower(label) NOT LIKE 'ephemeral%' AND label <> 'debugger-token' AND label NOT LIKE 'mcp-oauth-%' + AND NOT starts_with(label, 'embed_app:') + AND NOT starts_with(label, 'sdk_app:') + AND NOT starts_with(label, 'impersonation:') + AND NOT starts_with(label, 'cli-login:') )) RETURNING token_prefix", req.label.as_deref(), @@ -3284,6 +4055,12 @@ async fn leave_workspace( ) -> Result { forbid_job_token_account_destruction(&authed)?; let mut tx = db.begin().await?; + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("u/{}", authed.username), + ) + .await?; sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", &w_id, @@ -3406,12 +4183,63 @@ async fn get_all_runnables( #[derive(Deserialize, Debug, Clone)] pub struct LoginUserInfo { pub email: Option, + /// OIDC `email_verified` claim where the provider sends one. + #[serde(default, deserialize_with = "deserialize_lenient_bool")] + pub email_verified: Option, pub name: Option, pub company: Option, pub preferred_username: Option, pub displayName: Option, } +/// Some providers (Cognito among them) send `email_verified` as the strings "true"/"false"; +/// a strict bool would reject their whole userinfo document and break login. +fn deserialize_lenient_bool<'de, D: serde::Deserializer<'de>>( + d: D, +) -> std::result::Result, D::Error> { + Ok(match Option::::deserialize(d)? { + Some(serde_json::Value::Bool(b)) => Some(b), + Some(serde_json::Value::String(s)) => match s.trim().to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + }, + _ => None, + }) +} + +#[cfg(test)] +mod login_user_info_tests { + use super::LoginUserInfo; + + fn email_verified(json: &str) -> Option { + serde_json::from_str::(json) + .unwrap() + .email_verified + } + + #[test] + fn email_verified_accepts_bool_and_stringified_bool() { + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":true}"#), + Some(true) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"true"}"#), + Some(true) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"false"}"#), + Some(false) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"maybe"}"#), + None + ); + assert_eq!(email_verified(r#"{"email":"a@b"}"#), None); + } +} + #[derive(Serialize)] struct InstanceUsernameInfo { username: String, @@ -3541,6 +4369,9 @@ async fn overwrite_global_users( require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; let mut tx = db.begin().await?; + // Replaces the account table, so — unlike the paths that remove one account — it deliberately + // does not call `delete_drafts_of_email`: the addresses are about to be reinstated, and + // dropping every draft on the instance to restore accounts would be pure collateral. sqlx::query!("DELETE FROM password") .execute(&mut *tx) .await?; @@ -3710,3 +4541,23 @@ async fn request_password_reset( } // NOTE: reset_password is in windmill-api (depends on users_oss::hash_password EE dispatch) + +#[cfg(test)] +mod tests { + use super::*; + + /// Stored hashes outlive the hashing crate: every instance still holds hashes minted by + /// older argon2 releases, and an upgrade that stopped reading them locks their users out. + #[test] + fn verifies_a_hash_minted_by_an_older_argon2() { + // The seeded admin hash from migration 20220508150023, m=4096,t=3,p=1. + let seeded = "$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I"; + + assert!(Argon2::default() + .verify_password(b"changeme", seeded) + .is_ok()); + assert!(Argon2::default() + .verify_password(b"not-the-password", seeded) + .is_err()); + } +} diff --git a/backend/windmill-api-users/src/users_oss.rs b/backend/windmill-api-users/src/users_oss.rs index a42cce8405..a1eedd8563 100644 --- a/backend/windmill-api-users/src/users_oss.rs +++ b/backend/windmill-api-users/src/users_oss.rs @@ -26,3 +26,13 @@ pub async fn impersonate_service_account( "Service accounts require Windmill Enterprise Edition".to_string(), )) } + +#[cfg(not(feature = "private"))] +pub(crate) async fn portal_cloud_trial_login( + _email: &str, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::FeatureUnavailable( + "Starting a pre-approved trial from Windmill Cloud requires Windmill Enterprise Edition" + .to_string(), + )) +} diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 6ee55ed003..0102f1b033 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -19,6 +19,7 @@ use windmill_common::{ db::UserDB, error::JsonResult, jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE}, + queue_metrics::{read_queue_metrics_series, QueueMetricsSeries}, utils::{paginate, Pagination}, worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE}, workspaces::workspace_with_fork_ancestors, @@ -38,6 +39,8 @@ pub fn global_service() -> Router { ) .route("/get_default_tags", get(get_default_tags)) .route("/queue_metrics", get(get_queue_metrics)) + .route("/queue_metrics_series", get(get_queue_metrics_series)) + .route("/queue_status", get(get_queue_status)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -270,10 +273,16 @@ async fn get_queue_metrics( ) -> JsonResult> { require_devops_role(&db, &authed).await?; + // The API declares every `value` a number, so a climbing delay, stored as its head's wait + // start, is returned as the delay at the time of its sample. let queue_metrics = sqlx::query_as!( QueueMetric, "WITH queue_metrics as ( - SELECT id, value, created_at + SELECT id, created_at, + CASE WHEN jsonb_typeof(value) = 'object' + THEN to_jsonb(EXTRACT(EPOCH FROM created_at) - (value->>'since')::numeric) + ELSE value + END AS value FROM metrics WHERE id LIKE 'queue_%' AND created_at > now() - interval '14 day' @@ -289,6 +298,88 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct QueueMetricsSeriesQuery { + window_secs: Option, +} + +const QUEUE_METRICS_DEFAULT_WINDOW_SECS: i64 = 24 * 3600; +/// Retention of queue metrics, past which there is nothing left to read. +const QUEUE_METRICS_MAX_WINDOW_SECS: i64 = 14 * 24 * 3600; + +async fn get_queue_metrics_series( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_devops_role(&db, &authed).await?; + + let window = query + .window_secs + .unwrap_or(QUEUE_METRICS_DEFAULT_WINDOW_SECS) + .clamp(60, QUEUE_METRICS_MAX_WINDOW_SECS); + Ok(Json(read_queue_metrics_series(&db, window as f64).await?)) +} + +#[derive(Serialize)] +struct QueueTagStatus { + tag: String, + /// Jobs due for more than 3 seconds that no worker has picked up. + waiting: u32, + /// How long the job the next pull would take has been waiting, in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + delay: Option, + running: i64, + /// Workers that pinged in the last minute and pull this tag. + workers: i64, +} + +/// Every tag with jobs waiting or running, read live from the queue. A backlog on a tag no live +/// worker pulls waits for one to start: a worker group scaling up from zero, or none at all for a +/// tag nobody serves. +async fn get_queue_status( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let backlog = windmill_common::queue::get_queue_stats(&db).await?; + let backlog_tags = backlog.keys().cloned().collect::>(); + // A job's tag is resolved before it is queued (per-workspace and dedicated worker tags + // included), and the pull matches it exactly against the worker's tags, so containment is + // exact here too. + let rows = sqlx::query!( + "WITH running AS ( + SELECT tag, count(*) AS n FROM v2_job_queue WHERE running = true GROUP BY tag + ) + SELECT t.tag AS \"tag!\", COALESCE(r.n, 0) AS \"running!\", + (SELECT count(*) FROM worker_ping w + WHERE w.ping_at > now() - interval '1 minute' AND w.custom_tags @> ARRAY[t.tag] + ) AS \"workers!\" + FROM (SELECT tag::text FROM running UNION SELECT unnest($1::text[])) t(tag) + LEFT JOIN running r ON r.tag = t.tag + ORDER BY t.tag", + &backlog_tags[..], + ) + .fetch_all(&db) + .await?; + + Ok(Json( + rows.into_iter() + .map(|row| { + let stat = backlog.get(&row.tag); + QueueTagStatus { + waiting: stat.map_or(0, |s| s.count), + delay: stat.map(|s| s.delay), + running: row.running, + workers: row.workers, + tag: row.tag, + } + }) + .collect(), + )) +} + async fn get_queue_counts( authed: ApiAuthed, Extension(db): Extension, @@ -345,11 +436,6 @@ async fn get_workspace_fairness_events( WITH capped AS ( SELECT timestamp, operation, resource, parameters FROM audit_partitioned - WHERE workspace_id = 'admins' - AND operation = 'workspace_fairness.capped' - UNION ALL - SELECT timestamp, operation, resource, parameters - FROM audit WHERE workspace_id = 'admins' AND operation = 'workspace_fairness.capped' ORDER BY timestamp DESC @@ -357,11 +443,6 @@ async fn get_workspace_fairness_events( ), uncapped AS ( SELECT timestamp, operation, resource, parameters FROM audit_partitioned - WHERE workspace_id = 'admins' - AND operation = 'workspace_fairness.uncapped' - UNION ALL - SELECT timestamp, operation, resource, parameters - FROM audit WHERE workspace_id = 'admins' AND operation = 'workspace_fairness.uncapped' ORDER BY timestamp DESC diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs new file mode 100644 index 0000000000..8e7b75e1e2 --- /dev/null +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -0,0 +1,274 @@ +//! What the workspace key rotation, the workspace storage settings and the AI session backup +//! routes (`windmill-api/src/ai_sessions.rs`) share about the backups: the store they live +//! in, and what a rotation or a storage change deletes. +//! +//! The backups are ciphertext under the workspace key and live under a prefix named by a +//! generation the rotation bumps (`workspace_settings.ai_sessions_backup_generation`) in the +//! transaction that commits the new key. A rotation does not re-key them: once committed, +//! the routes read and write under the new generation's prefix and answer with its number +//! (`backup_generation`; the storage identity, `storage_id`, names the storage and does not +//! change), so every browser marks its sync state stale and pushes its sessions whole again +//! there, and every older generation, which nothing writes to any +//! more, is deleted off the request at leisure. Sessions no browser holds any more are lost, +//! which a rotation (a rare operation) accepts in exchange for having no key but the current +//! one to read with and nothing to rewrite in place. A generation is never reused, so no +//! deletion, however late, can touch live objects; a rotation that fails before its commit +//! bumps nothing and deletes nothing; two rotations racing serialize on the key row. +//! +//! A workspace without storage of its own keeps its backups in the instance object store +//! instead, under the same layout and key, while `ai_sessions_instance_storage_fallback` +//! allows it. Configuring a storage for such a workspace bumps the generation in the +//! transaction that sets it, so everything the workspace left in any instance store sits +//! under a generation the routes never read again: a later return to the instance store, +//! whichever it is by then, starts from a newer one. That is what lets a storage change +//! delete the older generations from the instance store without fencing against what +//! happens next, and a browser retire a removal owed to an instance store once the +//! workspace's own storage answered. + +use std::sync::Arc; + +use futures::{StreamExt, TryStreamExt}; +use windmill_common::error::{Error, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::DB; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, +}; +use windmill_object_store::{ + object_store_error_to_error, object_store_location, ObjectStoreResource, +}; +use windmill_types::s3::LargeFileStorage; + +/// The root of every AI session backup key in a workspace's storage. +pub const ROOT: &str = "windmill_ai_sessions"; +/// The push body cap: no object written through the routes is larger. One that is was +/// planted by whoever holds the bucket's credentials, and is left unread. +pub const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; +/// The storage name the workspace's backups in the instance store count under in its +/// storage usage, next to `_default_` and the secondary storages. +pub const FALLBACK_STORAGE: &str = "_ai_sessions_fallback_"; + +const IO_CONCURRENCY: usize = 8; + +/// The prefix of one generation's objects: `windmill_ai_sessions/{w_id}/g{generation}/`. +pub fn generation_prefix(w_id: &str, generation: i64) -> String { + format!("{ROOT}/{w_id}/g{generation}") +} + +/// The prefix of everything the workspace ever backed up, whatever the generation. +fn workspace_prefix(w_id: &str) -> ObjectPath { + ObjectPath::from(format!("{ROOT}/{w_id}")) +} + +/// Names the storage the backups are in, by what locates its objects (endpoint, region, +/// bucket; never the credentials, which rotate), so a browser tells that its sync state was +/// recorded against another storage; the generation, answered alongside, tells it a +/// rotation happened in this one. +pub fn storage_id(resource: &ObjectStoreResource) -> String { + calculate_hash(&object_store_location(resource))[..16].to_string() +} + +/// Where a workspace's backups live: its primary storage, or the instance object store +/// standing in for it. +pub struct BackupStore { + pub store: Arc, + pub storage_id: String, + pub fallback: bool, +} + +/// The instance object store, for a workspace without storage of its own: loaded from +/// settings that say where its objects are, and not turned off by +/// `ai_sessions_instance_storage_fallback`, which is on unless set to false. Named like a +/// workspace storage, by that location, in a namespace of its own. Never on the Pro plan, +/// checked on every call: a store loaded before a switch to Pro stays loaded. Never in a +/// build without `private`, which has neither workspace storage nor the quota the fallback +/// counts toward. +/// +/// Authorizes nothing, and the store reaches every workspace's objects: the caller must have +/// authorized the user for the workspace and keep what it reads and writes under that +/// user's prefix in it, as the backup routes do. +pub async fn fallback_store(db: &DB) -> Result> { + #[cfg(not(feature = "private"))] + { + let _ = db; + Ok(None) + } + #[cfg(feature = "private")] + { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ) { + return Ok(None); + } + let Some((store, Some(location))) = + windmill_object_store::get_object_store_with_location().await + else { + return Ok(None); + }; + let setting = windmill_common::global_settings::load_value_from_global_settings( + db, + windmill_common::global_settings::AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING, + ) + .await?; + if matches!(setting, Some(serde_json::Value::Bool(false))) { + return Ok(None); + } + Ok(Some(BackupStore { + storage_id: calculate_hash(&format!("instance:{location}"))[..16].to_string(), + store, + fallback: true, + })) + } +} + +/// The workspace's primary storage, resolved without a caller: a rotation runs its deletion +/// off its own request, and the retention sweep off any. The caller must be the server +/// itself; nothing here checks who asks. +pub async fn primary_store(db: &DB, w_id: &str) -> Result> { + let Some(lfs_json) = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() else { + return Ok(None); + }; + let lfs: LargeFileStorage = serde_json::from_value(lfs_json) + .map_err(|e| Error::internal_err(format!("parsing large_file_storage: {e}")))?; + let resource_value = if matches!(lfs, LargeFileStorage::FilesystemStorage(_)) { + serde_json::Value::Null + } else { + let path = lfs.get_s3_resource_path(); + let path = path.strip_prefix("$res:").unwrap_or(path); + windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{path}")), + w_id, + db, + ) + .await? + }; + let resource = windmill_object_store::lfs_to_object_store_resource(&lfs, resource_value)?; + Ok(Some(BackupStore { + store: windmill_object_store::build_object_store_client(&resource).await?, + storage_id: storage_id(&resource), + fallback: false, + })) +} + +/// The store the workspace's backups live in, resolved without a caller. +async fn workspace_store(db: &DB, w_id: &str) -> Result> { + if let Some(primary) = primary_store(db, w_id).await? { + return Ok(Some(primary)); + } + fallback_store(db).await +} + +/// The generation an object key sits under, `None` for a key of no generation (an older +/// layout), which counts as older than any. +fn generation_of(w_id: &str, key: &ObjectPath) -> Option { + key.as_ref() + .strip_prefix(&format!("{ROOT}/{w_id}/g"))? + .split('/') + .next()? + .parse() + .ok() +} + +/// Deletes, as the listing streams, every object of the workspace's backups in the store +/// from a generation older than `current`. +async fn delete_older(store: &Arc, w_id: &str, current: i64) -> Result<()> { + store + .list(Some(&workspace_prefix(w_id))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { + return Ok(()); + } + match store.delete(&meta.location).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + }) + .await +} + +/// Deletes, off the request, every object of the workspace's backups from a generation +/// older than `current`, once the rotation that made `current` the generation has +/// committed: nothing writes there any more but a push that resolved its prefix before the +/// commit, junk the browser's next push of that session rewrites under the current prefix, +/// as is anything a deletion cut short left behind. For the rotation route, which +/// authorized its caller as a superadmin. +pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { + tokio::spawn(async move { + let store = match workspace_store(&db, &w_id).await { + Ok(Some(store)) => store.store, + Ok(None) => return, + Err(e) => { + tracing::warn!("older AI session backups of {w_id} left in place: {e:#}"); + return; + } + }; + match delete_older(&store, &w_id, current).await { + Ok(()) => { + tracing::info!("deleted the AI session backups of {w_id} older than g{current}") + } + Err(e) => tracing::warn!("deleting the older AI session backups of {w_id}: {e:#}"), + } + }); +} + +/// Deletes, off the request, what the workspace's backups left in the instance store under +/// a generation older than `current`, the one a storage settings change committed. Nothing +/// reads there: the routes use the workspace's own storage, or, back in the instance store, +/// `current` or a newer generation, since configuring a storage over the fallback bumped +/// it. So it runs whatever the storage is now and whatever the setting says (copies from +/// when it was on may be there), and a deletion that is slow, cut short or overtaken by a +/// later change deletes nothing live. For the storage settings route, which authorized its +/// caller as a workspace admin. +pub(crate) fn spawn_delete_fallback(w_id: String, current: i64) { + tokio::spawn(async move { + let Some(instance) = windmill_object_store::get_object_store().await else { + return; + }; + match delete_older(&instance, &w_id, current).await { + Ok(()) => tracing::info!( + "deleted the AI session backups of {w_id} older than g{current} from the instance store" + ), + Err(e) => tracing::warn!( + "deleting the AI session backups of {w_id} from the instance store: {e:#}" + ), + } + }); +} + +/// The bytes of the workspace's backups in the instance store, for its storage usage while +/// it has no storage of its own (once it has one nothing writes there, and the change +/// deleted what was): `None` when it has one, when there is no instance store, or when +/// there is nothing, so no empty usage entry shows up. Whether the setting is on or off, +/// since copies from when it was on may be there. +/// +/// Authorizes nothing: for the storage usage recount, which reports a total for the +/// workspace it was run for and hands out nothing it read. +pub async fn fallback_bytes(db: &DB, w_id: &str) -> Result> { + let has_storage = sqlx::query_scalar!( + r#"SELECT large_file_storage IS NOT NULL AS "has_storage!" FROM workspace_settings WHERE workspace_id = $1"#, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if has_storage { + return Ok(None); + } + let Some(instance) = windmill_object_store::get_object_store().await else { + return Ok(None); + }; + let mut total: i64 = 0; + let mut stream = instance.list(Some(&workspace_prefix(w_id))); + while let Some(meta) = stream.next().await { + total += meta.map_err(object_store_error_to_error)?.size as i64; + } + Ok((total > 0).then_some(total)) +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs new file mode 100644 index 0000000000..04dc3d4e80 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -0,0 +1,2034 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Ownership and grants on the objects of an instance data table. +//! +//! [`datatable_permissions`](crate::datatable_permissions) decides who may connect as which role; +//! this decides what each role may then touch. Every change is a real `GRANT`, `REVOKE`, +//! `ALTER ... OWNER TO` or `ALTER DEFAULT PRIVILEGES`, so Postgres is what enforces it. +//! +//! Reading is open to anyone who reaches the data table. Planning and applying are for those who +//! administer it — admins of the workspace that governs it, and superadmins. All of it is +//! Enterprise Edition ([`crate::datatable_acl_oss`]). + +use std::collections::BTreeMap; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{get, post}, + Json, Router, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use tokio_postgres::error::{DbError, SqlState}; +use tokio_postgres::AsyncMessage; + +use windmill_api_auth::ApiAuthed; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::datatable_roles::{ + lock_role_catalog, quote_ident, read_role_catalog, read_role_catalog_tx, DatatableRoleCatalog, + ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, +}; +use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; +use windmill_common::workspaces::{resolve_governing_datatable, DataTable, GoverningDatatable}; +use windmill_common::{PgDatabase, DB}; + +use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_governing_datatable}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/datatable_acl/{datatable_name}", get(get_datatable_acl)) + .route( + "/datatable_acl/{datatable_name}/plan", + post(plan_datatable_acl), + ) + .route( + "/datatable_acl/{datatable_name}/apply", + post(apply_datatable_acl), + ) +} + +/// What a read or a change is about. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AclTarget { + /// The data table's own database — where the privilege to create schemas lives. + Database, + Schema { + schema: String, + }, + Table { + schema: String, + table: String, + }, +} + +impl AclTarget { + /// The schema the target is in, absent for the database itself. + pub(crate) fn schema(&self) -> Option<&str> { + match self { + AclTarget::Database => None, + AclTarget::Schema { schema } => Some(schema), + AclTarget::Table { schema, .. } => Some(schema), + } + } + + /// What it is called in a message. + pub(crate) fn label(&self, dbname: &str) -> String { + match self { + AclTarget::Database => dbname.to_string(), + AclTarget::Schema { schema } => schema.clone(), + AclTarget::Table { schema, table } => format!("{schema}.{table}"), + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclTargetQuery { + kind: String, + schema: Option, + table: Option, +} + +impl TryFrom for AclTarget { + type Error = Error; + fn try_from(q: AclTargetQuery) -> Result { + match (q.kind.as_str(), q.schema, q.table) { + ("database", _, _) => Ok(AclTarget::Database), + ("schema", Some(schema), _) => Ok(AclTarget::Schema { schema }), + ("table", Some(schema), Some(table)) => Ok(AclTarget::Table { schema, table }), + ("schema" | "table", None, _) => { + Err(Error::BadRequest("This target needs a schema".to_string())) + } + ("table", _, None) => Err(Error::BadRequest( + "A table target needs a table".to_string(), + )), + (kind, _, _) => Err(Error::BadRequest(format!("Unknown ACL target '{kind}'"))), + } + } +} + +/// Where a set of privileges applies, relative to the target. +/// +/// `Future*` covers what does not exist yet: those become `ALTER DEFAULT PRIVILEGES`, which only +/// binds objects created by the roles it names. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum GrantScope { + /// The target itself — the database, the schema, or the table. + Target, + AllTables, + AllSequences, + AllFunctions, + FutureTables, + FutureSequences, + FutureFunctions, +} + +impl GrantScope { + pub(crate) fn is_future(&self) -> bool { + matches!( + self, + GrantScope::FutureTables | GrantScope::FutureSequences | GrantScope::FutureFunctions + ) + } +} + +/// A change to plan. One at a time: each is confirmed against its own SQL. +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AclChange { + /// Hand the target — and, for a schema, everything already in it but an extension's members, + /// which stay with the extension — to another role. + SetOwner { + role: String, + }, + Grant { + role: String, + privileges: Vec, + scope: GrantScope, + }, + Revoke { + role: String, + privileges: Vec, + scope: GrantScope, + /// Objects inside the target, empty for the target itself. `ON ALL TABLES` grants read + /// back per object, so they are revoked per object — and the same privileges on several + /// of them are revoked together. + #[serde(default)] + objects: Vec, + }, +} + +impl AclChange { + /// The role the change is about, as the editor names it. + fn role(&self) -> &str { + match self { + AclChange::SetOwner { role } + | AclChange::Grant { role, .. } + | AclChange::Revoke { role, .. } => role, + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclChangeRequest { + pub target: AclTarget, + pub change: AclChange, + /// The statements the plan showed. An apply runs only those: it plans again and refuses if the + /// result differs. + #[serde(default)] + pub statements: Option>, +} + +/// An object inside a schema, named the way `REVOKE ... ON ` needs it. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct AclObject { + pub name: String, + /// `TABLE`, `SEQUENCE`, `FUNCTION`, `PROCEDURE` or `TYPE`: what the object is. + /// [`object_keyword`] turns it into the keyword a revoke takes; a type has none. + pub kind: String, + /// A routine is identified by its argument types, not by its name: two `f` in one schema are + /// two objects. Absent for everything else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub args: Option, +} + +/// A grant as the database has it, under the role names the editor uses. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclGrant { + /// A data table role's name, `admin` for `custom_instance_user`, else the raw Postgres role + /// (`PUBLIC` included). + pub grantee: String, + pub privileges: Vec, + /// `None` for the target itself, else the object inside it. + #[serde(skip_serializing_if = "Option::is_none")] + pub object: Option, + /// `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES` (or, on the database, `SCHEMAS`) when this + /// is a default privilege, which applies to objects that do not exist yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub future: Option, + /// Where the grant comes from, each role once: who granted it, or for a default privilege the + /// role whose future objects it covers. A revoke of some of the privileges takes them back from + /// every source that gave them. + pub sources: Vec, +} + +/// One role a grant comes from. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclSource { + /// Under the same names as [`AclGrant::grantee`]. + pub role: String, + /// What `role` gave of the grant's privileges. A revoke is held back only by a source out of + /// reach that gave some of what it takes back. + pub privileges: Vec, + /// Whether this data table's connection can take back what `role` gave: on an object only the + /// owner's grants when it acts for the owner, or else its own (`grant_source!`); for a default + /// privilege, a creating role it acts for. What a source out of reach gave is not revocable + /// from here; privileges only other sources gave still are. + pub reachable: bool, +} + +#[derive(Serialize, Debug)] +pub struct DatatableAclInfo { + /// Under the same names as [`AclGrant::grantee`]. + pub owner: String, + /// The roles a change may name: `admin`, then every role of the instance catalog. Only for a + /// caller who may change anything, as the catalog is in the permissions drawer. + pub roles: Vec, + /// Whether this caller may plan and apply changes: they administer the data table, on an + /// edition that has the planner. + pub editable: bool, + /// Whether the data table is a clone, whose grants stay as they were copied from its source. + pub clone: bool, + /// Whether the server is Postgres 17 or later, which added the `MAINTAIN` table privilege. + pub supports_maintain: bool, + /// The database the target lives in, which no target carries itself. + pub dbname: String, + pub grants: Vec, + /// What the target holds that is a target of its own: a database's schemas, a schema's tables. + pub children: Vec, +} + +#[derive(Serialize, Debug)] +pub struct AclPlan { + pub statements: Vec, + pub warnings: Vec, +} + +/// The Postgres role a role name stands for. A data table role is a login named exactly like the +/// role, so this is the identity — except `admin`, which is `custom_instance_user`. +/// +/// Anything else is refused, never resolved to some default: every statement a plan writes names +/// the role it is about. +pub(crate) fn pg_role_of(name: &str, catalog: &DatatableRoleCatalog) -> Result { + if name == ADMIN_DATATABLE_ROLE { + return Ok(CUSTOM_INSTANCE_USER.to_string()); + } + if catalog.values().any(|r| r.name == name) { + return Ok(name.to_string()); + } + Err(Error::BadRequest(format!( + "'{name}' is not a data table role of this instance" + ))) +} + +/// The reverse of [`pg_role_of`], for display. A role that is not a data table role reads back as +/// itself. +pub(crate) fn role_name_of(pg_role: &str) -> String { + if pg_role == CUSTOM_INSTANCE_USER { + ADMIN_DATATABLE_ROLE.to_string() + } else { + pg_role.to_string() + } +} + +/// Every role a change may name: `admin` first, then the catalog. +fn role_names(catalog: &DatatableRoleCatalog) -> Vec { + let mut names: Vec = catalog.values().map(|r| r.name.clone()).collect(); + names.sort(); + names.insert(0, ADMIN_DATATABLE_ROLE.to_string()); + names +} + +fn ensure_instance(governing: &GoverningDatatable) -> Result<()> { + if governing.is_instance() { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Data table '{}' is backed by a Postgres resource, so its access is managed on that \ + server directly. Only a data table on the Windmill instance's own database has data \ + table roles to grant to.", + governing.name + ))) +} + +/// The data table's `admin` connection, and the notices Postgres sends on it. +/// +/// Authorization: connects as `custom_instance_user` with the instance's own credentials and checks +/// nothing. Callers MUST have authorized the request first — a request about to be refused must +/// not get as far as this connection. +async fn connect_as_admin_unchecked( + db: &DB, + governing: &GoverningDatatable, +) -> Result<( + tokio_postgres::Client, + mpsc::UnboundedReceiver, + String, +)> { + ensure_instance(governing)?; + // Built from the authorized entry, never by resolving the settings again: a save in between + // could point the entry at a resource on another server and back, and this connection would + // then alter a database the later checks of the entry never see. + let mut pg = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + pg.dbname = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database") + .resource_path + .clone(); + pg.user = Some(CUSTOM_INSTANCE_USER.to_string()); + pg.password = Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); + let dbname = pg.dbname.clone(); + let (client, notices) = connect_with_notices(db, &pg).await?; + Ok((client, notices, dbname)) +} + +/// A connection to `pg`, and the notices Postgres sends on it — which is where a grant or revoke +/// that changed nothing is reported ([`execute_acl_statements`]). +pub(crate) async fn connect_with_notices( + db: &DB, + pg: &PgDatabase, +) -> Result<(tokio_postgres::Client, mpsc::UnboundedReceiver)> { + let (client, connection) = pg.connect(Some(db)).await?; + Ok(( + client, + drive_with_notices(connection, windmill_common::TokioPgConnection::poll_message), + )) +} + +type PollMessage = + fn( + &mut C, + &mut std::task::Context<'_>, + ) -> std::task::Poll>>; + +/// Drive `connection` in the background with `poll`, forwarding its notices. +pub(crate) fn drive_with_notices( + mut connection: C, + poll: PollMessage, +) -> mpsc::UnboundedReceiver { + // Unbounded: the driver must never wait on the receiver, which only drains once the statement + // the driver is carrying has completed. + let (notices_tx, notices) = mpsc::unbounded_channel(); + tokio::spawn(async move { + loop { + match std::future::poll_fn(|cx| poll(&mut connection, cx)).await { + Some(Ok(AsyncMessage::Notice(notice))) => { + let _ = notices_tx.send(notice); + } + Some(Ok(_)) => {} + Some(Err(e)) => { + tracing::error!("Datatable ACL connection error: {e}"); + break; + } + None => break, + } + } + }); + notices +} + +/// Run `statements` in order on `tx`, failing on the first that errors or that Postgres only warns +/// about. A privilege the connection cannot pass on is a warning to Postgres (`01007` / `01006`), +/// which then carries on having changed nothing; returning drops the transaction, rolling back +/// everything before it. +/// +/// Authorization: none. Runs `statements` as the connection `tx` is on; callers MUST have authorized +/// changing that database's access, and built the statements themselves. +pub(crate) async fn execute_acl_statements( + tx: &tokio_postgres::Transaction<'_>, + notices: &mut mpsc::UnboundedReceiver, + statements: &[String], +) -> Result<()> { + while notices.try_recv().is_ok() {} + for statement in statements { + tx.batch_execute(statement).await.map_err(|e| { + Error::ExecutionErr(format!( + "Failed to run `{statement}`: {}", + pg_error_message(&e) + )) + })?; + while let Ok(notice) = notices.try_recv() { + if *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_GRANTED + || *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_REVOKED + { + return Err(Error::ExecutionErr(format!( + "`{statement}` did not take effect ({}), so nothing was applied", + notice.message() + ))); + } + } + } + Ok(()) +} + +/// An object whose ownership follows the schema's. +#[derive(Debug, PartialEq)] +pub(crate) struct OwnedObject { + /// The keyword `ALTER ... OWNER TO` takes for this kind of object. + pub(crate) keyword: &'static str, + /// How Postgres names the object (`pg_identify_object`): schema-qualified, quoted where + /// needed, with a routine's arguments or an operator class's access method. It goes into the + /// statement as it is. + pub(crate) identity: String, +} + +/// Everything a schema's change of owner takes along, in schema `$1`, as (kind, identity, owner +/// oid), the first two as `pg_identify_object` gives them. The plan, the check of what this +/// connection may move and the check after the move all read this one list, so what is moved and +/// what is checked cannot differ. +/// +/// Read from what depends on the schema rather than catalog by catalog, so no kind of object is +/// left out by omission; array types, row types and indexes depend on another object instead. +/// Left out: an object's internal parts (a range type's constructors and multirange, an identity +/// column's sequence), a sequence tied to a column (it follows its table, and refuses an owner of +/// its own), an extension's members, and what has no `ALTER ... OWNER` at all (an extension, a text +/// search parser or template). Postgres records no owner for the bootstrap superuser, oid 10. +macro_rules! schema_owned_objects { + () => { + "SELECT o.type AS kind, o.identity, COALESCE(s.refobjid, 10::oid) AS owner + FROM pg_depend d + CROSS JOIN LATERAL pg_identify_object(d.classid, d.objid, 0) o + LEFT JOIN pg_shdepend s + ON s.dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND s.classid = d.classid AND s.objid = d.objid AND s.deptype = 'o' + WHERE d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n' + AND d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND d.classid <> ALL(ARRAY['pg_extension'::regclass, 'pg_ts_parser'::regclass, + 'pg_ts_template'::regclass]::oid[]) + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = d.classid AND x.objid = d.objid AND x.objsubid = 0 + AND (x.deptype IN ('i', 'e') + OR (x.deptype = 'a' AND d.classid = 'pg_class'::regclass + AND x.refobjsubid <> 0)))" + }; +} +#[allow(unused_imports)] +pub(crate) use schema_owned_objects; + +/// The keyword `ALTER ... OWNER TO` takes for a kind of object, as `pg_identify_object` names the +/// kind. A kind missing here is refused rather than skipped, which would leave it behind. +pub(crate) fn owned_keyword(kind: &str) -> Option<&'static str> { + Some(match kind { + "table" => "TABLE", + "view" => "VIEW", + "materialized view" => "MATERIALIZED VIEW", + "sequence" => "SEQUENCE", + "foreign table" => "FOREIGN TABLE", + "type" => "TYPE", + "function" | "procedure" | "aggregate" => "ROUTINE", + "collation" => "COLLATION", + "conversion" => "CONVERSION", + "operator" => "OPERATOR", + "operator class" => "OPERATOR CLASS", + "operator family" => "OPERATOR FAMILY", + "statistics object" => "STATISTICS", + "text search dictionary" => "TEXT SEARCH DICTIONARY", + "text search configuration" => "TEXT SEARCH CONFIGURATION", + _ => return None, + }) +} + +async fn read_owned_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + concat!( + "SELECT kind, identity FROM (", + schema_owned_objects!(), + ") o ORDER BY kind, identity" + ), + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + rows.into_iter() + .map(|row| { + let kind: &str = row.get(0); + let identity: String = row.get(1); + match owned_keyword(kind) { + Some(keyword) => Ok(OwnedObject { keyword, identity }), + None => Err(Error::BadRequest(format!( + "{identity} is a {kind}, whose owner cannot be changed from here, and schema \ + {schema} would change hands without it. Move it to another schema first." + ))), + } + }) + .collect() +} + +/// Default privileges set database-wide (no `IN SCHEMA`), as `(defaclrole, defaclobjtype, grantee, +/// privilege_type)`. Postgres stores such an entry as a whole acl, the creator's own privileges +/// included, so only what goes beyond the built-in default is a grant. It applies in every schema +/// on top of the schema's own defaults, which cannot take it back. +macro_rules! database_wide_defaults { + () => { + "SELECT d.defaclrole, d.defaclobjtype, a.grantee, a.privilege_type + FROM pg_default_acl d, aclexplode(d.defaclacl) a + WHERE d.defaclnamespace = 0 + AND NOT EXISTS ( + SELECT 1 FROM aclexplode(acldefault( + CASE d.defaclobjtype WHEN 'S' THEN 's' ELSE d.defaclobjtype END, d.defaclrole)) x + WHERE x.grantee = a.grantee AND x.privilege_type = a.privilege_type)" + }; +} + +/// `TABLES`, `SEQUENCES`, `FUNCTIONS` or `TYPES`, as a `defaclobjtype` names them. +fn default_objects_keyword(objtype: &str) -> &'static str { + match objtype { + "r" => "TABLES", + "S" => "SEQUENCES", + "T" => "TYPES", + _ => "FUNCTIONS", + } +} + +/// What a schema's owner holds on what gets created there later. A change of owner hands the new +/// owner the same and takes these back: otherwise every former owner keeps reaching whatever the +/// other roles create there. +#[derive(Debug, PartialEq)] +pub(crate) struct FormerOwnerDefaults { + pub(crate) pg_role: String, + /// (creating role, `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES`), one per default privilege + /// it holds. + pub(crate) defaults: Vec<(String, &'static str)>, +} + +/// The default privileges schema `schema`'s owner holds there — `None` when it holds none, or is +/// `new_owner` already. Refused when one was set by a role this connection cannot act for: only +/// a member of the creating role may change its defaults, so the revoke would fail at apply. Also +/// refused when the owner holds defaults set database-wide, which no change to this schema takes +/// back. +async fn read_former_owner_defaults( + client: &tokio_postgres::Client, + schema: &str, + new_owner: &str, +) -> Result> { + let database_wide = client + .query_opt( + concat!( + "SELECT pg_get_userbyid(n.nspowner), pg_get_userbyid(g.defaclrole), + g.defaclobjtype::text + FROM pg_namespace n, (", + database_wide_defaults!(), + ") g + WHERE n.nspname = $1 AND g.grantee = n.nspowner + AND g.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3 + LIMIT 1" + ), + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the database-wide default privileges: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = database_wide { + let pg_role: String = row.get(0); + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges set database-wide by {}, which no default of schema \ + {schema} takes back, so it would keep reaching what is created here after the change \ + of owner. Revoke them database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + role_name_of(&creator), + quote_ident(&creator), + default_objects_keyword(row.get(2)), + quote_ident(&pg_role) + ))); + } + + let rows = client + .query( + "SELECT DISTINCT pg_get_userbyid(n.nspowner), pg_get_userbyid(d.defaclrole), + d.defaclobjtype::text, pg_has_role(d.defaclrole, 'USAGE') + FROM pg_namespace n + JOIN pg_default_acl d ON d.defaclnamespace = n.oid + CROSS JOIN LATERAL aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND a.grantee = n.nspowner + -- What the owner gives itself is about the objects it creates, not about owning + -- the schema, so a move leaves it alone. + AND a.grantee <> d.defaclrole + AND d.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3", + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the default privileges of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + let Some(first) = rows.first() else { + return Ok(None); + }; + let pg_role: String = first.get(0); + let plural = |row: &tokio_postgres::Row| default_objects_keyword(row.get(2)); + if let Some(row) = rows.iter().find(|row| !row.get::<_, bool>(3)) { + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges in schema {schema} from {creator}, which this data \ + table's connection cannot act for, so they would outlive the change of owner. \ + Revoke them as {creator} first: ALTER DEFAULT PRIVILEGES FOR ROLE {} IN SCHEMA {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + quote_ident(&creator), + quote_ident(schema), + plural(row), + quote_ident(&pg_role) + ))); + } + Ok(Some(FormerOwnerDefaults { + defaults: rows + .iter() + .map(|row| (row.get::<_, String>(1), plural(row))) + .collect(), + pg_role, + })) +} + +/// What the catalog holds that a plan depends on, read before planning so the planner stays pure. +#[derive(Debug, Default, PartialEq)] +pub(crate) struct CatalogFacts { + /// Every role that may create objects here, but the one the change is about. + pub(crate) other_pg_roles: Vec, + /// For a schema's change of owner: what moves along with it. + pub(crate) existing_objects: Vec, + /// For a schema's change of owner: the defaults it takes back from the owner it replaces. + pub(crate) former_owner: Option, + /// For a revoke: each grant it takes back. + pub(crate) revoked_grants: Vec, +} + +/// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the +/// target itself when `None`), or for a default privilege on what `source` creates later. +#[derive(Debug, PartialEq)] +pub(crate) struct RevokedGrant { + pub(crate) object: Option, + /// The role that made the grant: its grantor, or the creating role of a default privilege. + pub(crate) source: String, + pub(crate) privileges: Vec, +} + +/// An `aclexplode` row's source and whether this connection can take back what it gave, on an +/// object owned by `$owner`. A REVOKE speaks for the grantor Postgres picks itself — the owner, +/// when the connection acts for the owner, and otherwise the connection — and `GRANTED BY` names +/// nobody else, so a grant any other role made stays whatever the connection runs. +macro_rules! grant_source { + ($owner:literal) => { + concat!( + "pg_get_userbyid(a.grantor), a.grantor = CASE WHEN pg_has_role(", + $owner, + ", 'USAGE') THEN ", + $owner, + " ELSE (SELECT oid FROM pg_roles WHERE rolname = current_user) END" + ) + }; +} + +/// What relation `$2` of schema `$1` grants role `$3`, as (source, whether this connection can +/// take it back, privilege). +const RELATION_GRANTS: &str = concat!( + "SELECT ", + grant_source!("c.relowner"), + ", a.privilege_type + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)" +); + +/// The grants a revoke takes back, read from the catalog rather than from the request: one per +/// object and source that gave `pg_role` any of `privileges`. Refused when a source's grant is out +/// of this connection's reach (`grant_source!`, or for a default privilege a creating role it does +/// not act for): the revoke would leave that grant in place. +async fn read_revoked_grants( + client: &tokio_postgres::Client, + dbname: &str, + target: &AclTarget, + scope: GrantScope, + objects: &[AclObject], + privileges: &[String], + pg_role: &str, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read what the revoke takes back: {}", + pg_error_message(&e) + )) + }; + let wanted: Vec = privileges.iter().map(|p| p.to_uppercase()).collect(); + // (object, how it reads in a refusal, one row per source and privilege) + let mut read = Vec::new(); + match (scope, target) { + (scope, AclTarget::Schema { schema }) if scope.is_future() => { + let (objtype, plural) = match scope { + GrantScope::FutureTables => ("r", "tables"), + GrantScope::FutureSequences => ("S", "sequences"), + _ => ("f", "functions"), + }; + let database_wide = client + .query( + concat!( + "SELECT pg_get_userbyid(g.defaclrole), g.privilege_type FROM (", + database_wide_defaults!(), + ") g + WHERE g.defaclobjtype::text = $1 + AND g.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1, 2" + ), + &[&objtype, &pg_role], + ) + .await + .map_err(read_error)?; + let mut still_granted: BTreeMap> = BTreeMap::new(); + for row in database_wide { + let privilege: String = row.get(1); + if wanted.contains(&privilege) { + still_granted.entry(row.get(0)).or_default().push(privilege); + } + } + if let Some((creator, taken)) = still_granted.into_iter().next() { + return Err(Error::BadRequest(format!( + "{} also receives {} on {plural} {} creates through a default privilege set \ + database-wide, which no default of schema {schema} takes back. Revoke it \ + database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} REVOKE {} ON {} \ + FROM {}", + role_name_of(pg_role), + taken.join(", "), + role_name_of(&creator), + quote_ident(&creator), + taken.join(", "), + default_objects_keyword(objtype), + quote_ident(pg_role) + ))); + } + let rows = client + .query( + "SELECT pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE'), + a.privilege_type + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND d.defaclobjtype::text = $2 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)", + &[schema, &objtype, &pg_role], + ) + .await + .map_err(read_error)?; + read.push(( + None, + format!("{plural} created later in schema {schema}"), + rows, + )); + } + (GrantScope::Target, _) if objects.is_empty() => { + let rows = match target { + AclTarget::Database => { + client + .query( + concat!( + "SELECT ", + grant_source!("d.datdba"), + ", a.privilege_type + FROM pg_database d, + aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $1)" + ), + &[&pg_role], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + concat!( + "SELECT ", + grant_source!("n.nspowner"), + ", a.privilege_type + FROM pg_namespace n, + aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2)" + ), + &[schema, &pg_role], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query(RELATION_GRANTS, &[schema, table, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push((None, target.label(dbname), rows)); + } + (GrantScope::Target, AclTarget::Schema { schema }) => { + for object in objects { + let rows = match object_keyword(&object.kind)? { + "ROUTINE" => { + client + .query( + concat!( + "SELECT ", + grant_source!("p.proowner"), + ", a.privilege_type + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND p.proname = $2 + AND pg_get_function_identity_arguments(p.oid) = $3 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $4)" + ), + &[ + schema, + &object.name, + &object.args.as_deref().unwrap_or(""), + &pg_role, + ], + ) + .await + } + _ => { + client + .query(RELATION_GRANTS, &[schema, &object.name, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push(( + Some(object.clone()), + format!("{} {schema}.{}", object.kind.to_lowercase(), object.name), + rows, + )); + } + } + // Every other scope and target is the planner's to refuse. + _ => {} + } + + let mut revoked = Vec::new(); + for (object, label, rows) in read { + let mut by_source: BTreeMap)> = BTreeMap::new(); + for row in rows { + let privilege: String = row.get(2); + if wanted.contains(&privilege) { + by_source + .entry(row.get(0)) + .or_insert_with(|| (row.get(1), vec![])) + .1 + .push(privilege); + } + } + for (source, (reachable, mut privileges)) in by_source { + if !reachable { + return Err(Error::BadRequest(format!( + "{} on {label} was granted to {} by {source}, and Postgres takes a grant \ + back only through the role that made it, which this data table's \ + connection cannot speak for here. Revoke it as {source}.", + privileges.join(", "), + role_name_of(pg_role), + ))); + } + privileges.sort(); + privileges.dedup(); + revoked.push(RevokedGrant { object: object.clone(), source, privileges }); + } + } + Ok(revoked) +} + +/// The keyword a `REVOKE ... ON` takes for one object, checked rather than interpolated: it lands +/// in SQL unquoted. +pub(crate) fn object_keyword(kind: &str) -> Result<&'static str> { + match kind.to_uppercase().as_str() { + "TABLE" | "VIEW" | "MATERIALIZED VIEW" | "FOREIGN TABLE" => Ok("TABLE"), + "SEQUENCE" => Ok("SEQUENCE"), + // `FUNCTION` names no procedure; `ROUTINE` names either. + "FUNCTION" | "PROCEDURE" | "ROUTINE" => Ok("ROUTINE"), + other => Err(Error::BadRequest(format!("Unknown object kind '{other}'"))), + } +} + +/// Every object of a schema, named the way the catalog names it. +async fn read_schema_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + "SELECT CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, c.relname, NULL::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + UNION ALL + SELECT CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, p.proname, + pg_get_function_identity_arguments(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = $1", + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + Ok(rows + .into_iter() + .map(|row| AclObject { kind: row.get(0), name: row.get(1), args: row.get(2) }) + .collect()) +} + +/// Replace the objects a revoke names with the catalog's own entry for each. +/// +/// A routine is identified by its argument types, and those go into the statement as written — +/// there is no quoting for them — so the request may name an object but never spell one: what +/// reaches the SQL is read back from Postgres. An object that resolves to nothing is refused rather +/// than dropped, since a revoke that silently covers less than it says is worse than an error. +async fn resolve_acl_objects( + client: &tokio_postgres::Client, + target: &AclTarget, + objects: &[AclObject], +) -> Result> { + if objects.is_empty() { + return Ok(vec![]); + } + let Some(schema) = target.schema() else { + return Err(Error::BadRequest( + "A database has no objects of its own to revoke on".to_string(), + )); + }; + let known = read_schema_objects(client, schema).await?; + objects + .iter() + .map(|requested| { + let keyword = object_keyword(&requested.kind)?; + known + .iter() + .find(|k| { + k.name == requested.name + && k.args == requested.args + && object_keyword(&k.kind).is_ok_and(|k| k == keyword) + }) + .cloned() + .ok_or_else(|| { + Error::NotFound(format!( + "'{}' is not an object of schema '{schema}'", + requested.name + )) + }) + }) + .collect() +} + +async fn read_owner(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let row = match target { + AclTarget::Database => { + client + .query_opt( + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query_opt( + // `public` is owned by `pg_database_owner`, a placeholder role whose membership + // is whoever owns the database — naming it back would say nothing, so resolve + // it to that owner. + "SELECT pg_get_userbyid(owner) FROM ( + SELECT CASE WHEN n.nspowner = (SELECT oid FROM pg_roles WHERE rolname = 'pg_database_owner') + THEN (SELECT d.datdba FROM pg_database d WHERE d.datname = current_database()) + ELSE n.nspowner END AS owner + FROM pg_namespace n WHERE n.nspname = $1 + ) o", + &[schema], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2", + &[schema, table], + ) + .await + } + } + .map_err(|e| Error::internal_err(format!("Failed to read the owner: {}", pg_error_message(&e))))?; + Ok(row.map(|row| row.get(0))) +} + +async fn read_children(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let rows = match target { + AclTarget::Database => { + client + .query( + "SELECT nspname::text FROM pg_namespace + WHERE nspname <> 'information_schema' AND nspname NOT LIKE 'pg\\_%' + ORDER BY nspname", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + "SELECT c.relname::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p']::\"char\"[]) + ORDER BY c.relname", + &[schema], + ) + .await + } + AclTarget::Table { .. } => return Ok(vec![]), + } + .map_err(|e| { + Error::internal_err(format!( + "Failed to list what the target holds: {}", + pg_error_message(&e) + )) + })?; + Ok(rows.into_iter().map(|row| row.get(0)).collect()) +} + +async fn get_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let target: AclTarget = query.try_into()?; + let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; + ensure_reaches_governing_datatable(&db, &w_id, &datatable_name, &governing, &authed).await?; + ensure_instance(&governing)?; + let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) + .await + .is_ok(); + let roles = if editable { + role_names(&read_role_catalog(&db).await?) + } else { + vec![] + }; + + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + let owner = read_owner(&client, &target) + .await? + .ok_or_else(|| Error::NotFound(format!("{} not found", target.label(&dbname))))?; + let grants = read_grants(&client, &target).await?; + let supports_maintain: bool = client + .query_one( + "SELECT current_setting('server_version_num')::int >= 170000", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the server version: {}", + pg_error_message(&e) + )) + })? + .get(0); + let children = read_children(&client, &target).await?; + + Ok(Json(DatatableAclInfo { + owner: role_name_of(&owner), + roles, + editable, + clone: governing.governor.is_some(), + supports_maintain, + dbname, + grants, + children, + })) +} + +async fn read_grants(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + // `aclexplode` turns an acl array into one row per (grantee, privilege); grantee 0 is PUBLIC, + // which has no name to resolve. A NULL acl is not "no access" but Postgres's built-in default — + // the owner holds everything and, on a routine, PUBLIC may EXECUTE — hence `acldefault`. The + // owner's own entries are left out: what it holds comes with ownership, which the owner shows, + // not with a grant a revoke here could take back. Each row ends with its source — the grantor, + // or a default privilege's creating role — and whether this connection can take back what it + // gave. + // Column-level grants (`pg_attribute.attacl`) are not supported yet: they are neither read here + // nor revocable from the editor. + let mut rows = match target { + AclTarget::Database => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("d.datdba"), + " FROM pg_database d, aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() AND a.grantee <> d.datdba" + ), + &[], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + // Default privileges set database-wide reach what is created in every + // schema, so they are the database's to show rather than any schema's. + concat!( + "SELECT CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END, + g.privilege_type, NULL::text, + CASE g.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' WHEN 'n' THEN 'SCHEMAS' + ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(g.defaclrole), pg_has_role(g.defaclrole, 'USAGE') + FROM (", + database_wide_defaults!(), + ") g" + ), + &[], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Schema { schema } => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("n.nspowner"), + " FROM pg_namespace n, aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 AND a.grantee <> n.nspowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, c.relname, NULL::text, + CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, + NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee <> c.relowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // Routines carry their own acl in `pg_proc`; without this a grant made here + // would vanish on the next read and could never be revoked back. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, p.proname, NULL::text, + CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, + pg_get_function_identity_arguments(p.oid), ", + grant_source!("p.proowner"), + " FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND a.grantee <> p.proowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // `USAGE` on a type is what lets a role use it in a column. Only a type the + // schema holds in its own right has an acl: an array, a row type and a + // multirange answer to their element, table or range. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, t.typname, NULL::text, 'TYPE', NULL::text, ", + grant_source!("t.typowner"), + " FROM pg_type t + JOIN pg_depend d ON d.classid = 'pg_type'::regclass AND d.objid = t.oid + AND d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n', + aclexplode(COALESCE(t.typacl, acldefault('T', t.typowner))) a + WHERE d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND a.grantee <> t.typowner + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = 'pg_type'::regclass AND x.objid = t.oid + AND x.objsubid = 0 AND x.deptype = 'i')" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // What a creating role set is taken back `FOR ROLE` that role, which only + // a role acting for it may do. + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, + CASE d.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE') + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1", + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Table { schema, table } => client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 AND a.grantee <> c.relowner" + ), + &[schema, table], + ) + .await + .map_err(grant_read_error)?, + }; + + // One row per privilege and source — and, for default privileges, per creating role. Fold them + // back into one entry per grantee and object that keeps every source and what each gave: a + // revoke takes a privilege back from every source that gave it. + let mut folded: BTreeMap< + ( + String, + Option<(String, String, Option)>, + Option, + ), + (Vec, BTreeMap)>), + > = BTreeMap::new(); + for row in rows.drain(..) { + let grantee: String = row.get(0); + let privilege: String = row.get(1); + let object: Option = row.get(2); + let future: Option = row.get(3); + let object_kind: Option = row.get(4); + let object_args: Option = row.get(5); + let source: String = row.get(6); + let reachable: bool = row.get(7); + let (privileges, sources) = folded + .entry(( + role_name_of(&grantee), + object.map(|name| { + ( + name, + object_kind.unwrap_or_else(|| "TABLE".to_string()), + object_args, + ) + }), + future, + )) + .or_default(); + sources + .entry(role_name_of(&source)) + .or_insert_with(|| (reachable, vec![])) + .1 + .push(privilege.clone()); + privileges.push(privilege); + } + Ok(folded + .into_iter() + .map(|((grantee, object, future), (mut privileges, sources))| { + privileges.sort(); + privileges.dedup(); + AclGrant { + grantee, + privileges, + object: object.map(|(name, kind, args)| AclObject { name, kind, args }), + future, + sources: sources + .into_iter() + .map(|(role, (reachable, mut privileges))| { + privileges.sort(); + privileges.dedup(); + AclSource { role, privileges, reachable } + }) + .collect(), + } + }) + .collect()) +} + +fn grant_read_error(e: tokio_postgres::Error) -> Error { + Error::internal_err(format!("Failed to read grants: {}", pg_error_message(&e))) +} + +/// Changing a data table's access is administering it. Checked in full before anything connects +/// with the instance's credentials. +async fn authorize_acl_change( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + ensure_governs_datatable(db, authed, w_id, &governing).await?; + ensure_instance(&governing)?; + Ok(governing) +} + +static APPLY_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +/// A role passes on only privileges it holds with grant option, and an instance database +/// provisioned before data table roles gave `custom_instance_user` none. Adds that option to its +/// database and `public` privileges, and nothing else: default privileges are left alone, since a +/// schema's change of owner is planned against them. Best-effort, as a grant it fails to enable is +/// refused when it runs. +async fn ensure_grant_options(client: &tokio_postgres::Client, db: &DB, dbname: &str) { + let held = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CONNECT WITH GRANT OPTION') + AND has_database_privilege(current_database(), 'CREATE WITH GRANT OPTION') + AND (to_regnamespace('public') IS NULL + OR (has_schema_privilege('public', 'USAGE WITH GRANT OPTION') + AND has_schema_privilege('public', 'CREATE WITH GRANT OPTION')))", + &[], + ) + .await + .is_ok_and(|row| row.get::<_, bool>(0)); + if held { + return; + } + if let Err(e) = grant_options_as_server(db, dbname).await { + tracing::warn!("Could not enable grant options on '{dbname}': {e}"); + } +} + +/// Only the database's owner, the server's own Postgres user, can hand out an option it holds. +async fn grant_options_as_server(db: &DB, dbname: &str) -> Result<()> { + let server = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + let creds = PgDatabase { dbname: dbname.to_string(), ..server }; + let (client, connection) = creds.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let role = quote_ident(CUSTOM_INSTANCE_USER); + let result = client + .batch_execute(&format!( + "GRANT CONNECT, CREATE ON DATABASE {} TO {role} WITH GRANT OPTION; + DO $$ BEGIN + IF to_regnamespace('public') IS NOT NULL THEN + GRANT USAGE, CREATE ON SCHEMA public TO {role} WITH GRANT OPTION; + END IF; + END $$;", + quote_ident(dbname) + )) + .await; + drop(client); + windmill_common::shutdown_pg_connection(join_handle).await?; + result.map_err(|e| { + Error::internal_err(format!( + "Failed to grant options on '{dbname}': {}", + pg_error_message(&e) + )) + }) +} + +/// Whether the governing entry an apply was authorized on is still the one in the settings, read +/// under the lock: a save in between could have pointed it at another database or changed its roles. +fn entry_unchanged(governing: &GoverningDatatable, entry_now: Option) -> bool { + let Some(Ok(now)) = entry_now.map(serde_json::from_value::) else { + return false; + }; + match ( + serde_json::to_value(&now), + serde_json::to_value(&governing.datatable), + ) { + (Ok(now), Ok(authorized)) => now == authorized, + _ => false, + } +} + +/// Plan one change against the catalog and the database as they are now. +async fn build_plan( + client: &tokio_postgres::Client, + dbname: &str, + catalog: &DatatableRoleCatalog, + target: &AclTarget, + change: &AclChange, +) -> Result { + let role = change.role(); + // `admin` is the login the data table itself reaches Postgres through, and the one every + // change here runs as: a revoke that lands leaves nothing able to grant it back. + if matches!(change, AclChange::Revoke { .. }) && role == ADMIN_DATATABLE_ROLE { + return Err(Error::BadRequest(format!( + "'{ADMIN_DATATABLE_ROLE}' is how this data table reaches its database; \ + its own access is not revocable from here" + ))); + } + let pg_role = pg_role_of(role, catalog)?; + let change = match change { + AclChange::Revoke { role, privileges, scope, objects } => AclChange::Revoke { + role: role.clone(), + privileges: privileges.clone(), + scope: *scope, + objects: resolve_acl_objects(client, target, objects).await?, + }, + change => change.clone(), + }; + // Default privileges are recorded per creating role, and a schema's new owner is kept in reach + // of what the others create there, so both are written for every role there is. + let other_pg_roles = role_names(catalog) + .iter() + .map(|name| pg_role_of(name, catalog)) + .filter(|r| r.as_ref().map_or(true, |r| *r != pg_role)) + .collect::>>()?; + let (existing_objects, former_owner) = match (&change, target) { + (AclChange::SetOwner { .. }, AclTarget::Schema { schema }) => ( + read_owned_objects(client, schema).await?, + read_former_owner_defaults(client, schema, &pg_role).await?, + ), + _ => (vec![], None), + }; + let revoked_grants = match &change { + AclChange::Revoke { privileges, scope, objects, .. } => { + read_revoked_grants( + client, dbname, target, *scope, objects, privileges, &pg_role, + ) + .await? + } + _ => vec![], + }; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some((object, owner)) = unmanaged_owner(client, target).await? { + return Err(Error::BadRequest(format!( + "{object} is owned by {owner}, which this data table's connection cannot act \ + for, so its owner cannot be changed from here" + ))); + } + } + let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let mut plan = + crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some(missing) = missing_owner_privilege(client, target, &pg_role).await? { + plan.warnings.push(format!( + "{role} does not have {missing}, which Postgres requires of a new owner, so this \ + will be refused. Grant it first." + )); + } + } + Ok(plan) +} + +/// Postgres only hands an object to a role that could have created it: a table to one with +/// `CREATE` on its schema, a schema to one with `CREATE` on the database. +async fn missing_owner_privilege( + client: &tokio_postgres::Client, + target: &AclTarget, + pg_role: &str, +) -> Result> { + let (row, missing) = match target { + AclTarget::Table { schema, .. } => ( + client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'CREATE')", + &[&pg_role, schema], + ) + .await, + format!("CREATE on schema {schema}"), + ), + AclTarget::Schema { .. } => ( + client + .query_one( + "SELECT has_database_privilege($1::name, current_database(), 'CREATE')", + &[&pg_role], + ) + .await, + "CREATE on the database".to_string(), + ), + AclTarget::Database => return Ok(None), + }; + let has: bool = row + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the new owner's privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + Ok((!has).then_some(missing)) +} + +/// The first thing a change of owner would move that this connection cannot act for, with its +/// owner. Postgres lets only a member of the current owner move an object, and every change runs as +/// `custom_instance_user`, so an object it does not hold the owner of — `public`, owned by the +/// database's owner, above all — is refused here rather than at apply. Running as the instance's +/// own user instead would reach objects Windmill never created. +async fn unmanaged_owner( + client: &tokio_postgres::Client, + target: &AclTarget, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read who owns what the change moves: {}", + pg_error_message(&e) + )) + }; + match target { + AclTarget::Database => Ok(None), + AclTarget::Table { schema, table } => { + let row = client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 AND NOT pg_has_role(c.relowner, 'USAGE')", + &[schema, table], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| (format!("{schema}.{table}"), row.get(0)))) + } + AclTarget::Schema { schema } => { + let row = client + .query_opt( + concat!( + "SELECT ord, identity, pg_get_userbyid(owner) FROM ( + SELECT 0 AS ord, NULL::text AS identity, n.nspowner AS owner + FROM pg_namespace n WHERE n.nspname = $1 + UNION ALL + SELECT 1, identity, owner FROM (", + schema_owned_objects!(), + ") m + ) o WHERE NOT pg_has_role(owner, 'USAGE') ORDER BY ord, identity LIMIT 1" + ), + &[schema], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| { + let label = row + .get::<_, Option>(1) + .unwrap_or_else(|| format!("schema {schema}")); + (label, row.get(2)) + })) + } + } +} + +async fn plan_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + let catalog = read_role_catalog(&db).await?; + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + Ok(Json( + build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?, + )) +} + +async fn apply_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> Result { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let confirmed = req.statements.as_ref().ok_or_else(|| { + Error::BadRequest( + "An apply runs exactly the statements its plan showed; plan the change first" + .to_string(), + ) + })?; + // Everything that needs the pool happens before the locks: once `tx` holds them, a second pool + // connection could wait forever on a pool that concurrent applies, queued on the same locks, + // have exhausted. + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + // Applies queue on an instance-wide lock while each holds a direct connection to the instance's + // Postgres; unbounded, the queue alone could exhaust its connection limit. One at a time per + // server, and the ones waiting hold no connection at all. + let _slot = APPLY_SLOT + .acquire() + .await + .map_err(|e| Error::internal_err(format!("ACL apply slot closed: {e}")))?; + let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + ensure_grant_options(&client, &db, &dbname).await; + + // Held until the change is committed: a role renamed or dropped meanwhile would change what + // the plan names, and a settings save could move the entry onto another database. Taken in the + // same order as the permissions save, so the two cannot deadlock. + let mut tx = db.begin().await?; + lock_role_catalog(&mut tx).await?; + let entry_now = sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&governing.workspace_id) + .bind(&governing.name) + .fetch_optional(&mut *tx) + .await? + .flatten(); + let catalog = read_role_catalog_tx(&mut tx).await?; + + let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; + if !entry_unchanged(&governing, entry_now) || &plan.statements != confirmed { + return Err(Error::BadRequest( + "The data table or its roles changed since this was planned, so it would no longer \ + run what was confirmed. Plan it again." + .to_string(), + )); + } + + // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two + // different roles. + let pg_tx = client.transaction().await.map_err(|e| { + Error::internal_err(format!( + "Failed to open a transaction on the data table: {}", + pg_error_message(&e) + )) + })?; + execute_acl_statements(&pg_tx, &mut notices, &plan.statements).await?; + + // A schema's objects were listed before the transaction opened; one committed since would stay + // with its old owner. One created while this transaction is still open can still slip past, as + // Postgres has no lock that holds creation in a schema back. That is benign: it stays with its + // creator, like anything created there later, and moving that table fixes it. + if let (AclChange::SetOwner { role }, AclTarget::Schema { schema }) = (&req.change, &req.target) + { + let new_owner = pg_role_of(role, &catalog)?; + let straggler = pg_tx + .query_opt( + concat!( + "SELECT identity FROM (", + schema_owned_objects!(), + ") m WHERE owner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1 LIMIT 1" + ), + &[schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to check what the schema holds: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = straggler { + return Err(Error::BadRequest(format!( + "{} appeared while this ran and would keep its old owner, so nothing was \ + applied. Plan it again.", + row.get::<_, String>(0) + ))); + } + } + + let target_label = req.target.label(&dbname); + audit_log( + &mut *tx, + &authed, + "workspaces.datatable_acl", + ActionKind::Update, + &governing.workspace_id, + Some(&governing.name), + Some( + [ + ("target", target_label.as_str()), + ("change", change_kind(&req.change)), + ("role", req.change.role()), + ] + .into(), + ), + ) + .await?; + pg_tx.commit().await.map_err(|e| { + Error::internal_err(format!( + "Failed to commit the changes: {}", + pg_error_message(&e) + )) + })?; + tx.commit().await?; + + windmill_common::feature_usage::log_feature_usage( + "datatable", + "acl_applied", + change_kind(&req.change), + ); + + Ok(format!("Updated access on {target_label}")) +} + +/// What kind of change, never what it named: the telemetry key and the audit's summary. +fn change_kind(change: &AclChange) -> &'static str { + match change { + AclChange::SetOwner { .. } => "owner", + AclChange::Grant { scope, .. } | AclChange::Revoke { scope, .. } if scope.is_future() => { + "default_privileges" + } + AclChange::Grant { .. } => "grant", + AclChange::Revoke { .. } => "revoke", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::datatable_roles::InstanceDatatableRole; + + #[test] + fn a_role_is_its_own_postgres_role_except_admin() { + let catalog: DatatableRoleCatalog = BTreeMap::from([( + "role1".to_string(), + InstanceDatatableRole { name: "analytics".to_string(), enabled: true, pwd: None }, + )]); + assert_eq!( + pg_role_of("admin", &catalog).unwrap(), + "custom_instance_user" + ); + assert_eq!(pg_role_of("analytics", &catalog).unwrap(), "analytics"); + assert_eq!(role_name_of("custom_instance_user"), "admin"); + assert_eq!(role_name_of("analytics"), "analytics"); + // A catalog id, a name the catalog lacks, or the admin login spelled out never stands for + // some other role. + for unknown in ["role1", "operator", "custom_instance_user", "PUBLIC", ""] { + assert!( + matches!(pg_role_of(unknown, &catalog), Err(Error::BadRequest(_))), + "{unknown}" + ); + } + } + + /// A connection to the test's own database, the way the handlers reach a data table's. + async fn catalog_client(pool: &sqlx::PgPool) -> tokio_postgres::Client { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + config.dbname(pool.connect_options().get_database().unwrap()); + let (client, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + client + } + + /// What a revoke takes back is read from the catalog, per object and source, and only the + /// privileges it asks for: the planner renders exactly this, and refuses an empty read. + #[sqlx::test(migrations = false)] + async fn a_revoke_reads_back_only_what_it_asks_for(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + // A predefined role, so that nothing is granted outside the test's own database. + client + .batch_execute( + "CREATE SCHEMA granted; + CREATE TABLE granted.g (id int); + GRANT SELECT, INSERT ON granted.g TO pg_read_all_data;", + ) + .await + .unwrap(); + let target = AclTarget::Table { schema: "granted".to_string(), table: "g".to_string() }; + let revoked = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(revoked.len(), 1, "{revoked:?}"); + assert_eq!(revoked[0].object, None); + assert_eq!(revoked[0].privileges, ["SELECT"]); + let held_none = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["update".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert!(held_none.is_empty(), "{held_none:?}"); + // Each source says what it gave: that, and not the whole row, is what a revoke of some of + // its privileges is held back by. + let grants = read_grants( + &client, + &AclTarget::Schema { schema: "granted".to_string() }, + ) + .await + .unwrap(); + let on_g = grants + .iter() + .find(|g| { + g.grantee == "pg_read_all_data" && g.object.as_ref().is_some_and(|o| o.name == "g") + }) + .unwrap(); + assert_eq!(on_g.sources.len(), 1, "{:?}", on_g.sources); + assert_eq!(on_g.sources[0].privileges, ["INSERT", "SELECT"]); + assert!(on_g.sources[0].reachable); + } + + /// A default privilege set database-wide applies in every schema on top of the schema's own, + /// which cannot take it back: the database shows it, and neither a schema's revoke of it nor a + /// change of owner away from its grantee may go ahead as if it were gone. + #[sqlx::test(migrations = false)] + async fn a_schema_cannot_take_back_a_database_wide_default(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA owned AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA owned + GRANT SELECT, INSERT ON TABLES TO pg_read_all_data; + CREATE SCHEMA creators AUTHORIZATION pg_monitor; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_monitor + GRANT SELECT ON TABLES TO pg_read_all_stats;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + + let grants = read_grants(&client, &AclTarget::Database).await.unwrap(); + let database_wide = grants + .iter() + .find(|g| g.grantee == "pg_read_all_data" && g.future.as_deref() == Some("TABLES")) + .unwrap_or_else(|| panic!("{grants:?}")); + assert_eq!(database_wide.privileges, ["SELECT"]); + // A database-wide entry also holds its creator's own privileges, which come with creating + // and are no grant: neither a row nor a refusal may stem from them. + assert!( + !grants + .iter() + .any(|g| g.future.is_some() && (g.grantee == creator || g.grantee == "pg_monitor")), + "{grants:?}" + ); + assert_eq!( + database_wide.sources.len(), + 1, + "{:?}", + database_wide.sources + ); + assert_eq!(database_wide.sources[0].role, creator); + + let schema = AclTarget::Schema { schema: "owned".to_string() }; + let still_granted = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await; + assert!( + matches!(&still_granted, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{still_granted:?}" + ); + let schema_only = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["insert".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(schema_only.len(), 1, "{schema_only:?}"); + assert_eq!(schema_only[0].privileges, ["INSERT"]); + + let moved = read_former_owner_defaults(&client, "owned", "pg_write_all_data").await; + assert!( + matches!(&moved, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{moved:?}" + ); + let moved_from_creator = + read_former_owner_defaults(&client, "creators", "pg_write_all_data").await; + assert!( + matches!(moved_from_creator, Ok(None)), + "{moved_from_creator:?}" + ); + } + + /// Defaults on types go with a schema's owner like the other kinds: once a database-wide + /// default takes PUBLIC's USAGE on types away, they are all that reaches a new type. + #[sqlx::test(migrations = false)] + async fn a_schemas_former_owner_defaults_include_types(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA typed AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA typed GRANT USAGE ON TYPES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_read_all_data IN SCHEMA typed + GRANT SELECT ON TABLES TO pg_read_all_data;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + let former = read_former_owner_defaults(&client, "typed", "pg_write_all_data") + .await + .unwrap(); + assert_eq!( + former, + Some(FormerOwnerDefaults { + pg_role: "pg_read_all_data".to_string(), + defaults: vec![(creator, "TYPES")], + }) + ); + } + + /// A kind of object the list misses stays with its old owner while the schema changes hands, + /// which only a real catalog shows. + #[sqlx::test(migrations = false)] + async fn a_schemas_owner_change_takes_every_object_in_it(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA moved; + CREATE TABLE moved.t (id serial PRIMARY KEY, a int, b int); + CREATE STATISTICS moved.st ON a, b FROM moved.t; + CREATE TABLE moved.pt (id int) PARTITION BY RANGE (id); + CREATE TABLE moved.pt1 PARTITION OF moved.pt FOR VALUES FROM (0) TO (10); + CREATE TYPE moved.r AS RANGE (subtype = float8); + CREATE TYPE moved.pair AS (a int, b int); + CREATE TYPE moved.mood AS ENUM ('ok'); + CREATE DOMAIN moved.tags AS text[]; + CREATE COLLATION moved.coll (provider = libc, locale = 'C');", + ) + .await + .unwrap(); + let owned = read_owned_objects(&client, "moved").await.unwrap(); + // Not the serial's sequence, the range's constructors and multirange, or any array or row + // type: each follows the object it belongs to. + assert_eq!( + owned + .iter() + .map(|o| (o.keyword, o.identity.as_str())) + .collect::>(), + [ + ("COLLATION", "moved.coll"), + ("STATISTICS", "moved.st"), + ("TABLE", "moved.pt"), + ("TABLE", "moved.pt1"), + ("TABLE", "moved.t"), + ("TYPE", "moved.mood"), + ("TYPE", "moved.pair"), + ("TYPE", "moved.r"), + ("TYPE", "moved.tags"), + ] + ); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl_oss.rs b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs new file mode 100644 index 0000000000..59eb8cdbe0 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the ACL planner comes from: the enterprise one, or a refusal. +//! +//! Data table roles are an Enterprise Edition feature, and so is everything here — reading who +//! owns what included. `private` alone is not that edition — community builds carry it — so the +//! planner is behind `enterprise` as well. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_acl_ee::plan_statements; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_datatable_acl_available() -> windmill_common::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use { + crate::datatable_acl::{AclChange, AclPlan, AclTarget, CatalogFacts}, + windmill_common::{datatable_roles_oss::datatable_roles_unavailable, error::Result}, +}; + +/// Checked first by every ACL route, before anything is read or connected to. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn ensure_datatable_acl_available() -> Result<()> { + Err(datatable_roles_unavailable()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn plan_statements( + _target: &AclTarget, + _change: &AclChange, + _dbname: &str, + _pg_role: &str, + _facts: &CatalogFacts, +) -> Result { + Err(datatable_roles_unavailable()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_clone.rs b/backend/windmill-api-workspaces/src/datatable_clone.rs new file mode 100644 index 0000000000..a57a2216e5 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_clone.rs @@ -0,0 +1,566 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Copying a data table's database for the fork being created. +//! +//! The fork request makes its copies before it writes the fork: each is created, filled and — for a +//! data table under roles — given the source's owners and grants. What each copy was made from +//! stays in the request ([`MadeCopy`]), so the fork's transaction checks it against the source as it +//! is then, and a fork that fails drops the instance copies it made ([`drop_copies_after`]) and names +//! the others, which live on servers of the workspace's own. + +use std::collections::BTreeSet; + +use windmill_api_auth::ApiAuthed; +use windmill_common::datatable_roles::{lock_role_catalog, read_role_catalog_tx}; +use windmill_common::error::{pg_error_message, Error, Result}; +use windmill_common::utils::require_admin; +use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::workspaces::{ + get_datatable_resource_from_db_unchecked, DataTableDatabase, DataTableForkBehavior, + GoverningDatatable, +}; +use windmill_common::{PgDatabase, DB}; + +use crate::datatable_acl::connect_with_notices; +use crate::datatable_permissions::ensure_reaches_governing_datatable; +use crate::workspaces::{ + create_database_on_server, ensure_datatable_is_clonable, pg_dump_database, pg_import_dump, + DumpFile, PgDumpOptions, +}; + +/// A database this request created and filled for one data table of the fork. +pub(crate) struct MadeCopy { + /// The data table's name, in the parent and in the fork. + pub(crate) name: String, + pub(crate) dbname: String, + pub(crate) behavior: DataTableForkBehavior, + /// The database the source resolved to when it was copied. + pub(crate) source_database: DataTableDatabase, + /// Whether the source's owners and grants were replayed into the copy: it was under roles. + pub(crate) replayed: bool, + /// For a resource-backed copy, the resource and variables its connection was resolved from. + pub(crate) connection: Option, +} + +/// What a resource-backed data table's connection is resolved from: its resource, and every +/// resource and variable that one references, as stored — and, for a secret kept in an external +/// backend, the value that backend holds. The connection is resolved from the snapshot itself +/// ([`ConnectionSnapshot::resolve`]), so it is exactly the one these rows describe. +#[derive(PartialEq)] +pub(crate) struct ConnectionSnapshot { + root: String, + resources: std::collections::BTreeMap>, + /// Stored value and whether it is secret. + variables: std::collections::BTreeMap>, + external_secrets: std::collections::BTreeMap, +} + +/// Read the [`ConnectionSnapshot`] of resource `resource_path` in workspace `w_id`. +/// +/// Authorization: none, and it holds secret values. Callers MUST have authorized using that +/// resource, and never disclose the snapshot or what it resolves to. +pub(crate) async fn connection_snapshot( + db: &DB, + conn: &mut sqlx::PgConnection, + w_id: &str, + resource_path: &str, +) -> Result { + let root = resource_path.trim_start_matches("$res:").to_string(); + let mut snapshot = ConnectionSnapshot { + root: root.clone(), + resources: Default::default(), + variables: Default::default(), + external_secrets: Default::default(), + }; + let mut pending = vec![root]; + while let Some(path) = pending.pop() { + if snapshot.resources.contains_key(&path) { + continue; + } + let value: Option = + sqlx::query_scalar("SELECT value FROM resource WHERE workspace_id = $1 AND path = $2") + .bind(w_id) + .bind(&path) + .fetch_optional(&mut *conn) + .await? + .flatten(); + let mut strings = vec![]; + collect_strings(value.as_ref(), &mut strings); + for reference in strings { + if let Some(var) = reference.strip_prefix("$var:") { + if snapshot.variables.contains_key(var) { + continue; + } + let row: Option<(String, bool)> = sqlx::query_as( + "SELECT value, is_secret FROM variable WHERE workspace_id = $1 AND path = $2", + ) + .bind(w_id) + .bind(var) + .fetch_optional(&mut *conn) + .await?; + if let Some((stored, true)) = &row { + if windmill_common::secret_backend::is_external_stored_value(stored) { + let secret = windmill_common::secret_backend::get_secret_value( + db, w_id, var, stored, + ) + .await?; + snapshot.external_secrets.insert(var.to_string(), secret); + } + } + snapshot.variables.insert(var.to_string(), row); + } else if let Some(res) = reference.strip_prefix("$res:") { + pending.push(res.to_string()); + } + } + snapshot.resources.insert(path, value); + } + Ok(snapshot) +} + +impl ConnectionSnapshot { + /// The connection these rows resolve to, as the data table's own resolution would: references + /// substituted, secrets decrypted. + pub(crate) async fn resolve(&self, db: &DB, w_id: &str) -> Result { + let root = self.resource(&self.root)?; + self.substitute(db, w_id, root, 0).await + } + + /// Whether the resource itself holds the connection's fields, which the fork repoints by setting + /// its `dbname`, rather than being a reference to another resource. + pub(crate) fn root_holds_connection(&self) -> bool { + self.resource(&self.root).is_ok_and(|v| v.is_object()) + } + + fn resource(&self, path: &str) -> Result<&serde_json::Value> { + self.resources + .get(path) + .and_then(|v| v.as_ref()) + .ok_or_else(|| Error::NotFound(format!("resource {path} does not exist"))) + } + + fn substitute<'a>( + &'a self, + db: &'a DB, + w_id: &'a str, + value: &'a serde_json::Value, + depth: usize, + ) -> std::pin::Pin> + Send + 'a>> + { + Box::pin(async move { + if depth > 32 { + return Err(Error::BadRequest( + "resource references nest too deeply".to_string(), + )); + } + Ok(match value { + serde_json::Value::Object(map) => { + let mut out = serde_json::Map::new(); + for (key, val) in map { + out.insert(key.clone(), self.substitute(db, w_id, val, depth).await?); + } + serde_json::Value::Object(out) + } + serde_json::Value::Array(items) => { + let mut out = vec![]; + for val in items { + out.push(self.substitute(db, w_id, val, depth).await?); + } + serde_json::Value::Array(out) + } + serde_json::Value::String(s) if s.starts_with("$res:") => { + let path = &s[5..]; + self.substitute(db, w_id, self.resource(path)?, depth + 1) + .await? + } + serde_json::Value::String(s) if s.starts_with("$var:") => { + let path = &s[5..]; + let (stored, secret) = self + .variables + .get(path) + .and_then(|v| v.as_ref()) + .ok_or_else(|| { + Error::NotFound(format!("variable {path} does not exist")) + })?; + serde_json::Value::String(match (secret, self.external_secrets.get(path)) { + (false, _) => stored.clone(), + (true, Some(external)) => external.clone(), + (true, None) => windmill_common::variables::decrypt( + &windmill_common::variables::build_crypt(db, w_id).await?, + stored.clone(), + ) + .map_err(|e| { + Error::internal_err(format!("Error decrypting variable {s}: {e}")) + })?, + }) + } + other => other.clone(), + }) + }) + } +} + +fn collect_strings<'a>(value: Option<&'a serde_json::Value>, out: &mut Vec<&'a str>) { + match value { + Some(serde_json::Value::String(s)) => out.push(s), + Some(serde_json::Value::Array(items)) => { + items.iter().for_each(|v| collect_strings(Some(v), out)) + } + Some(serde_json::Value::Object(map)) => { + map.values().for_each(|v| collect_strings(Some(v), out)) + } + _ => {} + } +} + +/// What a data table of the fork should be copied as. +pub(crate) struct CopyRequest<'a> { + pub(crate) name: &'a str, + pub(crate) dbname: &'a str, + pub(crate) behavior: DataTableForkBehavior, +} + +/// Check that `authed` may copy each data table of `parent_w_id` as asked, before anything is +/// created. +/// +/// Who may copy is what it was before data table roles — anyone for the schema, an admin of the +/// workspace for the rows — narrowed under roles to whoever may connect as one of them: even the +/// schema alone lists every table, which under roles only role holders can read in the parent. +pub(crate) async fn authorize_copies( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + requests: &[CopyRequest<'_>], +) -> Result<()> { + for request in requests { + match request.behavior { + DataTableForkBehavior::KeepOriginal => { + return Err(Error::BadRequest(format!( + "Data table '{}' is kept, which copies nothing", + request.name + ))) + } + DataTableForkBehavior::SchemaOnly => {} + DataTableForkBehavior::SchemaAndData => { + require_admin(authed.is_admin, &authed.username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Cloning schema and data is not available on cloud".to_string(), + )); + } + } + } + windmill_common::validate_dbname(request.dbname)?; + if !request.dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest(format!( + "Forked datatable database name '{}' must start with 'wm_fork_'", + request.dbname + ))); + } + let governing = ensure_datatable_is_clonable(db, parent_w_id, request.name).await?; + ensure_reaches_governing_datatable(db, parent_w_id, request.name, &governing, authed) + .await?; + if governing.datatable.permissions.is_some() { + crate::datatable_replay_oss::ensure_replay()?; + } + } + Ok(()) +} + +/// Make every copy in `requests`, in order, once [`authorize_copies`] allows them all. When one +/// fails, the copies already made are dropped before the error is returned. +pub(crate) async fn make_copies( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + requests: &[CopyRequest<'_>], +) -> Result> { + authorize_copies(db, authed, parent_w_id, requests).await?; + let mut copies = Vec::with_capacity(requests.len()); + for request in requests { + match make_copy(db, authed, parent_w_id, request).await { + Ok(copy) => copies.push(copy), + Err(e) => return Err(drop_copies_after(db, copies, e).await), + } + } + Ok(copies) +} + +/// Drop every copy, returning `error` — with what could not be dropped appended, since nothing +/// else will name those databases again. +pub(crate) async fn drop_copies_after(db: &DB, copies: Vec, error: Error) -> Error { + let mut stranded = Vec::new(); + for copy in copies { + if let Err(e) = drop_copy(db, ©.source_database, ©.dbname).await { + tracing::error!("Could not drop '{}' after a failed fork: {e}", copy.dbname); + stranded.push(format!("'{}' ({e})", copy.dbname)); + } + } + if stranded.is_empty() { + error + } else { + Error::ExecutionErr(format!( + "{error}. These databases created for the fork could not be dropped: {}", + stranded.join(", ") + )) + } +} + +async fn drop_copy(db: &DB, source_database: &DataTableDatabase, dbname: &str) -> Result<()> { + if source_database.resource_type + == windmill_common::workspaces::DataTableCatalogResourceType::Instance + { + match windmill_common::drop_unused_instance_database(db, dbname).await? { + windmill_common::Cleanup::Dropped => Ok(()), + windmill_common::Cleanup::InUse(users) => Err(Error::BadRequest(format!( + "kept, since workspaces {} now use it", + users.join(", ") + ))), + windmill_common::Cleanup::Waiter(pid) => Err(Error::BadRequest(format!( + "kept, since a request (pid {pid}) is still waiting to name it" + ))), + } + } else { + // On a server of the workspace's own, where a resource edited meanwhile can already name it + // and nothing locks such an edit: dropping it could take someone's data. + Err(Error::BadRequest( + "kept on its PostgreSQL server, where it may already be in use; drop it there once it \ + is not, before forking the same data table under this id again" + .to_string(), + )) + } +} + +async fn make_copy( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + request: &CopyRequest<'_>, +) -> Result { + let governing = ensure_datatable_is_clonable(db, parent_w_id, request.name).await?; + let source_database = governing.datatable.database.clone().ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + request.name + )) + })?; + let is_instance = governing.is_instance(); + // Resolved from the snapshot, not read again: the connection the copy is made on is then + // exactly what these rows describe, which the fork's own clone of them is checked against. + let (server, connection): (PgDatabase, Option) = if is_instance { + let server = serde_json::from_value( + get_datatable_resource_from_db_unchecked(db, parent_w_id, request.name).await?, + ) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {e}")))?; + (server, None) + } else { + let snapshot = connection_snapshot( + db, + &mut *db.acquire().await?, + &governing.workspace_id, + &source_database.resource_path, + ) + .await?; + if !snapshot.root_holds_connection() { + return Err(Error::BadRequest(format!( + "Data table '{}' uses resource '{}', which only refers to another resource: the \ + fork's copy of it could not be pointed at the copied database. Point the data \ + table at the resource holding the connection, then fork again.", + request.name, source_database.resource_path + ))); + } + let server = serde_json::from_value(snapshot.resolve(db, &governing.workspace_id).await?) + .map_err(|e| { + Error::internal_err(format!("Failed to parse database credentials: {e}")) + })?; + (server, Some(snapshot)) + }; + + // Dumped before the database exists, so a source that cannot be read leaves nothing behind. + // Ownership never carries over: the restore runs as the target's connection user. Grants do, + // except on the instance, where the replay below is what reproduces them. + let dump = pg_dump_database( + &server, + PgDumpOptions { + schema_only: request.behavior == DataTableForkBehavior::SchemaOnly, + no_owner: true, + no_acl: is_instance, + ..Default::default() + }, + ) + .await?; + + if is_instance { + windmill_common::create_custom_instance_database(db, request.dbname, "datatable").await?; + } else { + create_database_on_server(db, &server, request.dbname).await?; + } + + let target = PgDatabase { dbname: request.dbname.to_string(), ..server.clone() }; + let filled = fill( + db, + authed, + parent_w_id, + request.name, + &governing, + &server, + &target, + &dump, + ) + .await; + match filled { + Ok(replayed) => Ok(MadeCopy { + name: request.name.to_string(), + dbname: request.dbname.to_string(), + behavior: request.behavior, + source_database, + replayed, + connection, + }), + Err(e) => { + if let Err(drop_err) = drop_copy(db, &source_database, request.dbname).await { + tracing::error!( + "Could not drop '{}' after a failed copy of data table '{}': {drop_err}", + request.dbname, + request.name + ); + return Err(Error::ExecutionErr(format!( + "{e}. The database '{}' created for the copy could not be dropped: {drop_err}", + request.dbname + ))); + } + Err(e) + } + } +} + +/// Restore the dump into the new database and, for a data table under roles, replay the source's +/// owners and grants into it. Returns whether it replayed. +#[allow(clippy::too_many_arguments)] +async fn fill( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + name: &str, + governing: &GoverningDatatable, + source: &PgDatabase, + target: &PgDatabase, + dump: &DumpFile, +) -> Result { + pg_import_dump(target, dump).await?; + + // Held until the replay commits: a role renamed or dropped meanwhile would change what the + // replay names, and a settings save could move the source onto another database or put it under + // roles. Taken in the same order as the permissions save and the ACL apply. + // On a connection of its own: the checks below take theirs from the pool, and a lock holder + // drawn from the pool too could leave a small one with nothing to give them. + let database_url = windmill_common::get_database_url().await?; + let mut lock_holder = ::connect_with( + &database_url.connect_options().await?, + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to connect to the database: {e}")))?; + let mut tx = sqlx::Connection::begin(&mut lock_holder).await?; + lock_role_catalog(&mut tx).await?; + lock_settings_rows(&mut tx, parent_w_id, name).await?; + + // Everything so far was decided before the locks. + let now = ensure_datatable_is_clonable(db, parent_w_id, name).await?; + ensure_reaches_governing_datatable(db, parent_w_id, name, &now, authed).await?; + if !same_database(&now.datatable.database, &governing.datatable.database) { + return Err(Error::BadRequest(format!( + "Data table '{name}' moved to another database while it was being copied; fork again" + ))); + } + + let replayed = now.datatable.permissions.is_some(); + if replayed { + crate::datatable_replay_oss::ensure_replay()?; + let catalog = read_role_catalog_tx(&mut tx).await?; + // The replay leaves `CONNECT` to the catalog, and creating the database only tried to set + // it: a copy `PUBLIC` could still connect to would admit logins the source turns away. + windmill_common::datatable_roles::converge_connect_grants_with( + db, + &target.dbname, + &catalog, + ) + .await?; + let catalog_roles: BTreeSet = catalog.values().map(|r| r.name.clone()).collect(); + let (source, _source_notices) = connect_with_notices(db, source).await?; + let (mut target, mut notices) = connect_with_notices(db, target).await?; + // One transaction on the copy: a replay that stops halfway leaves objects owned by one role + // and granted as another. + let pg_tx = target.transaction().await.map_err(|e| { + Error::internal_err(format!( + "Failed to open a transaction on the copy: {}", + pg_error_message(&e) + )) + })?; + crate::datatable_replay_oss::replay_owners_and_grants( + &source, + &pg_tx, + &mut notices, + &catalog_roles, + ) + .await?; + pg_tx.commit().await.map_err(|e| { + Error::internal_err(format!( + "Failed to commit the replayed grants: {}", + pg_error_message(&e) + )) + })?; + } + tx.commit().await?; + Ok(replayed) +} + +/// Lock every settings row the resolution of data table `name` of `start_w_id` passes through, in +/// the order it passes them — each pointer, the entry that owns the database, and each workspace its +/// roles come from — and return them. Deleting or repointing any of them would leave a copy made +/// for the resolution answering for another one. +pub(crate) async fn lock_settings_rows( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + start_w_id: &str, + name: &str, +) -> Result> { + let mut path: Vec<(String, String)> = vec![]; + let (mut w_id, mut datatable) = (start_w_id.to_string(), name.to_string()); + loop { + if path.contains(&(w_id.clone(), datatable.clone())) || path.len() > 32 { + return Err(Error::BadRequest(format!( + "Data table '{name}' of workspace '{start_w_id}' resolves through a cycle" + ))); + } + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->$2 FROM workspace_settings + WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&w_id) + .bind(&datatable) + .fetch_optional(&mut **tx) + .await? + .flatten(); + path.push((w_id.clone(), datatable.clone())); + let entry: Option = + entry.and_then(|e| serde_json::from_value(e).ok()); + match entry.and_then(|e| e.reference.or(e.governed_by)) { + Some(next) => (w_id, datatable) = (next.workspace_id, next.datatable), + None => break, + } + } + Ok(path.into_iter().map(|(w_id, _)| w_id).collect()) +} + +pub(crate) fn same_database(a: &Option, b: &Option) -> bool { + match (a, b) { + (Some(a), Some(b)) => { + a.resource_type == b.resource_type && a.resource_path == b.resource_path + } + _ => false, + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index bf0b7dd213..df1fac203b 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -10,7 +10,10 @@ //! management, and the workspace-merge diff helper. Split out of `workspaces.rs` //! to keep that file focused on core workspace configuration. -use crate::workspaces::{pg_dump_database, ItemComparison}; +use crate::workspaces::{ + is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison, + PgDumpOptions, +}; use axum::{ extract::{Extension, Path, Query}, @@ -27,6 +30,7 @@ use windmill_api_auth::{require_super_admin, ApiAuthed}; use windmill_api_jobs::run_wait_result_internal; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; +use windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE; use windmill_common::db::UserDB; use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; use windmill_common::jobs::{JobPayload, RawCode}; @@ -35,7 +39,11 @@ use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, Debounci use windmill_common::scripts::ScriptLang; use windmill_common::users::username_to_permissioned_as; use windmill_common::worker::to_raw_value; -use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; +use windmill_common::worker::SqlAnnotations; +use windmill_common::workspaces::{ + ensure_can_use_datatable_role, ensure_datatable_admin_access, + get_datatable_resource_from_db_unchecked, resolve_governing_datatable, DatatableAccess, +}; use windmill_common::{PgDatabase, DB}; use windmill_git_sync::{ handle_deployment_metadata, handle_deployment_metadata_batch, DeployedObject, @@ -83,6 +91,79 @@ pub(crate) fn routes() -> Router { ) } +/// Refuse a migration whose role this caller may not use, before a job is pushed or a version +/// recorded. +/// +/// A migration that declares `-- role ` runs as that role, so the caller has to be one of its +/// tenants. One that declares none runs as `admin` and reaches every object in the database +/// whatever the roles grant, so it is for the admins of the workspace that governs the data table +/// — a fork can run a migration under a role it holds, never a migration under `admin`. +/// +/// The executor re-checks the role when it resolves the connection, so this is not the boundary. It +/// is what makes the refusal legible: which migration, and which role. +async fn ensure_migration_role_allowed( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, + sql: &str, + migration: &str, +) -> Result<()> { + let access = DatatableAccess::Authed(authed.to_authed_ref()); + match SqlAnnotations::datatable_role(sql)? { + Some(role) => { + ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, migration) + .await + } + None => ensure_datatable_admin_access(db, w_id, datatable_name, &access) + .await + .map_err(|e| { + Error::NotAuthorized(format!( + "{migration} declares no role, so it would run as admin. {e}" + )) + }), + } +} + +/// Refuse writing or deleting a migration definition unless this caller may run both its up and its +/// down. A definition is run later by whoever applies or rolls back pending migrations, as the role +/// it declares, so gating only the run would let anyone plant SQL for a data table's admin to run. +async fn ensure_may_edit_migration( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, + migration: &str, + code_up: &str, + code_down: Option<&str>, +) -> Result<()> { + // Admin access reaches everything a role can, so it edits any definition without reading its + // annotation, which one saved before annotations were parsed may no longer pass. A refusal + // there would leave such a definition impossible to delete or fix. + let access = DatatableAccess::Authed(authed.to_authed_ref()); + if ensure_datatable_admin_access(db, w_id, datatable_name, &access) + .await + .is_ok() + { + return Ok(()); + } + for sql in std::iter::once(code_up).chain(code_down) { + ensure_migration_role_allowed(db, w_id, datatable_name, authed, sql, migration).await?; + } + Ok(()) +} + +/// Operators run deployed code only, and a migration is SQL written or run by hand. Checked before +/// anything else, including the role checks: on a data table not under roles those pass everyone. +fn ensure_not_operator(authed: &ApiAuthed) -> Result<()> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot manage or run data table migrations".to_string(), + )); + } + Ok(()) +} + #[derive(Serialize)] struct AppliedMigration { version: i64, @@ -125,7 +206,18 @@ async fn datatable_database_arg( .await? .ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?; - Ok(to_raw_value(&format!("datatable://{datatable_name}"))) + // `?role=admin` rather than a bare reference, so a migration that declares no `-- role` runs + // as the connection that owns the schema instead of falling through to the data table's + // default role — which is what `ensure_migration_role_allowed` gated it as, and which is the + // only role a DDL statement can be expected to succeed under. A migration that does declare a + // role overrides this: the annotation wins over the reference. + // + // A legacy name containing `?` cannot be migrated through this reference: the appended query + // makes it neither an exact name nor a parseable one. Accepted on purpose, since such names can + // no longer be created and none are expected to carry migrations. + Ok(to_raw_value(&format!( + "datatable://{datatable_name}?role={ADMIN_DATATABLE_ROLE}" + ))) } /// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as @@ -381,6 +473,12 @@ async fn run_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + ensure_not_operator(&authed)?; + // Before the admin connection is opened at all: the bookkeeping below is created and read + // through it, so a caller no role covers must be refused here rather than after the fact. + crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) + .await?; + audit_log( &db, &authed, @@ -413,6 +511,16 @@ async fn run_datatable_migrations( let applied_versions = read_applied_versions_on_client(&client, &datatable_name).await?; + // How the user scoped the run, for the counter emitted on the first migration + // that lands below. + let scope = if query.only.is_some() { + "only" + } else if query.up_to.is_some() { + "up_to" + } else { + "all" + }; + let mut applied = Vec::new(); for m in migrations { if let Some(only) = query.only { @@ -427,6 +535,15 @@ async fn run_datatable_migrations( if applied_versions.contains(&m.timestamp) { continue; } + ensure_migration_role_allowed( + &db, + &w_id, + &datatable_name, + &authed, + &m.code_up, + &format!("Migration {} ({})", m.timestamp, m.name), + ) + .await?; run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) .await .map_err(|e| { @@ -450,6 +567,14 @@ async fn run_datatable_migrations( )) })?; applied.push(AppliedMigration { version: m.timestamp, name: m.name }); + // One event per run that moved the data table forward, emitted on the + // first migration that lands rather than after the loop: a later one + // failing returns early, and that run still advanced the data table. A + // run with nothing pending stays uncounted — it is the common outcome of + // opening the list and would drown out the runs that did something. + if applied.len() == 1 { + windmill_common::feature_usage::log_feature_usage("datatable", "migration_run", scope); + } } Ok(Json(RunDatatableMigrationsResult { applied })) @@ -485,6 +610,12 @@ async fn rollback_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + ensure_not_operator(&authed)?; + // Before the admin connection is opened at all: the bookkeeping below is created and read + // through it, so a caller no role covers must be refused here rather than after the fact. + crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) + .await?; + audit_log( &db, &authed, @@ -567,6 +698,16 @@ async fn rollback_datatable_migrations( )) })?; + ensure_migration_role_allowed( + &db, + &w_id, + &datatable_name, + &authed, + &code_down, + &format!("Migration {} ({})", version, definition.name), + ) + .await?; + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down) .await @@ -591,6 +732,12 @@ async fn rollback_datatable_migrations( )) })?; + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_rollback", + if query.only.is_some() { "only" } else { "last" }, + ); + Ok(Json(RollbackDatatableMigrationsResult { rolled_back: vec![RolledBackMigration { version, name: definition.name }], })) @@ -721,10 +868,15 @@ async fn read_applied_datatable_versions( /// List a data table's migrations annotated with whether each has been applied. async fn datatable_migrations_status( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { + // Reads `_wm_migrations` through the data table's admin connection, so it answers to the same + // question as running one: may you reach this data table at all. + crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) + .await?; + let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; if !enabled { return Ok(Json(DatatableMigrationsStatusResult { @@ -821,6 +973,8 @@ async fn enable_datatable_migrations( ) .await?; + windmill_common::feature_usage::log_feature_usage("datatable", "migrations_toggled", "on"); + Ok(format!( "Enabled migrations for data table {datatable_name}" )) @@ -889,6 +1043,8 @@ async fn disable_datatable_migrations( .await?; } + windmill_common::feature_usage::log_feature_usage("datatable", "migrations_toggled", "off"); + Ok(format!( "Disabled migrations for data table {datatable_name} and deleted its migrations" )) @@ -1094,9 +1250,20 @@ async fn create_datatable_migration( Path((w_id, datatable_name)): Path<(String, String)>, Json(payload): Json, ) -> JsonResult { + ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration '{}'", payload.name), + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; let mut tx = db.begin().await?; let timestamp = insert_datatable_migration_def( @@ -1131,6 +1298,8 @@ async fn create_datatable_migration( ) .await?; + windmill_common::feature_usage::log_feature_usage("datatable", "migration_created", "manual"); + Ok(Json(DatatableMigration { datatable: datatable_name, timestamp, @@ -1146,6 +1315,34 @@ async fn delete_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, ) -> Result { + ensure_not_operator(&authed)?; + let deleted_message = format!("Deleted migration {} from {}", timestamp, datatable_name); + + // A definition already gone is not an error: `wmill sync push` deletes whatever the repo + // dropped, which may no longer exist here. + let Some(definition) = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_optional(&db) + .await? + else { + return Ok(deleted_message); + }; + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", timestamp, definition.name), + &definition.code_up, + definition.code_down.as_deref(), + ) + .await?; + // Hold the run-serialization lock across the applied-check and the delete: a // run snapshots a migration's SQL before recording its version, so an // unserialized delete could race it and leave `_wm_migrations` pointing at a @@ -1173,19 +1370,46 @@ async fn delete_datatable_migration( ))); } + // Delete only the SQL the role check above cleared: it read the definition without the lock, + // and a rewrite that landed since was never checked. let deleted_name = sqlx::query_scalar!( "DELETE FROM datatable_migrations \ WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 \ + AND code_up = $4 AND code_down IS NOT DISTINCT FROM $5 \ RETURNING name", &w_id, &datatable_name, timestamp, + &definition.code_up, + definition.code_down.as_deref(), ) .fetch_optional(&db) .await?; // The definition is removed; runs may resume (audit/deploy metadata below // don't need the lock). drop(lock_client); + let Some(name) = deleted_name else { + // Nothing matched either because another delete got there first, which is this one's + // outcome too, or because the definition was rewritten. + let still_defined = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3)", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !still_defined { + return Ok(deleted_message); + } + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' changed while it was being deleted. Reload it and \ + retry.", + timestamp, datatable_name + ))); + }; audit_log( &db, @@ -1198,23 +1422,10 @@ async fn delete_datatable_migration( ) .await?; - // Only tally a change if a migration was actually deleted. - if let Some(name) = deleted_name { - record_datatable_migration_deployment( - &authed, - &db, - &w_id, - &datatable_name, - timestamp, - &name, - ) + record_datatable_migration_deployment(&authed, &db, &w_id, &datatable_name, timestamp, &name) .await?; - } - Ok(format!( - "Deleted migration {} from {}", - timestamp, datatable_name - )) + Ok(deleted_message) } #[derive(Deserialize)] @@ -1235,6 +1446,7 @@ async fn upsert_datatable_migration( Path((w_id, datatable_name)): Path<(String, String)>, Json(payload): Json, ) -> Result { + ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; @@ -1260,9 +1472,9 @@ async fn upsert_datatable_migration( // run-serialization lock across the applied-check and the write below so an // in-flight run can't record a version for the SQL we're about to overwrite. // Held until the end of the handler (well past the write). The exempt - // upserts need no lock: a new or unchanged one overwrites nothing, and one - // that only adds a down leaves the `code_up` a concurrent run is recording - // a version for untouched. + // upserts need no lock: a new one overwrites nothing, and one that only adds + // a down leaves the `code_up` a concurrent run is recording a version for + // untouched. let only_adds_down = existing.as_ref().is_some_and(|existing| { existing.name == payload.name && existing.code_up == payload.code_up @@ -1274,8 +1486,40 @@ async fn upsert_datatable_migration( && existing.code_up == payload.code_up && existing.code_down == payload.code_down }); + let upserted_message = format!( + "Upserted migration {} in {}", + payload.timestamp, datatable_name + ); + // A re-push of the definition as stored changes nothing, so it is not checked, written, audited, + // synced or counted. Writing it anyway would re-create a definition deleted since the read above. + if unchanged { + return Ok(upserted_message); + } + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", payload.timestamp, payload.name), + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; + // Replacing a definition removes the one stored, so it answers to the same check as a delete. + if let Some(existing) = existing.as_ref() { + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", payload.timestamp, existing.name), + &existing.code_up, + existing.code_down.as_deref(), + ) + .await?; + } let _run_lock = match existing.as_ref() { - Some(_) if !only_adds_down && !unchanged => { + Some(_) if !only_adds_down => { // Fail closed: if we can't lock/read the applied set (e.g. the // data-table database is temporarily unreachable), refuse the change // rather than risk overwriting a migration that has already run. @@ -1305,31 +1549,27 @@ async fn upsert_datatable_migration( _ => None, }; - // An exempt upsert judged the row from an unlocked read and then writes - // without the lock, so that whole read is re-tested here, where `ON CONFLICT - // DO UPDATE` re-reads the row under a row lock. Otherwise a request working - // from a stale definition silently reverts whatever changed in between — a - // second addition's down, or a locked rewrite of the up whose new SQL a run - // may already have recorded a version for. Reading no row at all is part of - // the premise: the equality is NULL when `$8` is, so a version created in the - // meantime is refused rather than overwritten. - let recheck_observed = _run_lock.is_none(); + // The role checks, and the exemptions from the lock, all judged the row from the unlocked read + // above, so that whole read is re-tested here, where `ON CONFLICT DO UPDATE` re-reads the row + // under a row lock. Otherwise a request working from a stale definition silently replaces + // whatever changed in between: SQL the role checks never saw, or a locked rewrite of the up + // whose new SQL a run may already have recorded a version for. Reading no row at all is part of + // the premise: the equality is NULL when `$7` is, so a version created in the meantime is + // refused rather than overwritten. let written = sqlx::query!( "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ VALUES ($1, $2, $3, $4, $5, $6) \ ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down \ - WHERE NOT $7 \ - OR (datatable_migrations.name = $8::text \ - AND datatable_migrations.code_up = $9::text \ - AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", + WHERE datatable_migrations.name = $7::text \ + AND datatable_migrations.code_up = $8::text \ + AND datatable_migrations.code_down IS NOT DISTINCT FROM $9::text", &w_id, &datatable_name, payload.timestamp, &payload.name, &payload.code_up, payload.code_down.as_deref(), - recheck_observed, existing.as_ref().map(|e| e.name.as_str()), existing.as_ref().map(|e| e.code_up.as_str()), existing.as_ref().and_then(|e| e.code_down.as_deref()), @@ -1368,10 +1608,17 @@ async fn upsert_datatable_migration( ) .await?; - Ok(format!( - "Upserted migration {} in {}", - payload.timestamp, datatable_name - )) + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_created", + if existing.is_none() { + "synced" + } else { + "edited" + }, + ); + + Ok(upserted_message) } /// Generate the first migration for a data table by snapshotting its current @@ -1384,6 +1631,16 @@ async fn generate_initial_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { + ensure_not_operator(&authed)?; + // Returns a `pg_dump` of the whole schema and writes into the data table's own bookkeeping, so + // it answers to the workspace that governs it rather than to whoever is asking. + ensure_datatable_admin_access( + &db, + &w_id, + &datatable_name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; validate_datatable_path_segment(&datatable_name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; @@ -1408,18 +1665,25 @@ async fn generate_initial_datatable_migration( let pg_db: PgDatabase = serde_json::from_value(db_resource) .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; - // Snapshot the schema, excluding Windmill's own migration bookkeeping table. - let dump_file = pg_dump_database(&pg_db, true, &["_wm_migrations"]).await?; + // Snapshot the schema without `_wm_migrations`, Windmill's own bookkeeping table, and + // without what a replay elsewhere cannot run: the replaying user owns none of this + // database's objects, and the grants Windmill plants in an instance database (`ALTER + // DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server. + let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?; + let dump_file = pg_dump_database( + &pg_db, + PgDumpOptions { + schema_only: true, + exclude_tables: &["_wm_migrations"], + no_owner: true, + no_acl, + }, + ) + .await?; let raw_dump = tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read schema dump: {}", e)))?; - // pg_dump emits psql meta-commands (\restrict / \unrestrict) that aren't - // valid SQL; drop them so the migration body can run via a plain query. - let code_up: String = raw_dump - .lines() - .filter(|line| !line.trim_start().starts_with('\\')) - .collect::>() - .join("\n"); + let code_up = strip_unreplayable_dump_lines(&raw_dump); // Record the definition first, then mark it installed. If marking fails we // delete the definition, so a failure leaves no phantom "initial" (rather @@ -1467,6 +1731,12 @@ async fn generate_initial_datatable_migration( ) .await?; + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_created", + "initial_snapshot", + ); + Ok(Json(DatatableMigration { datatable: datatable_name, timestamp, @@ -1541,9 +1811,21 @@ pub(crate) struct DatatableRename { pub(crate) to: String, } -async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result { +/// The database whose `_wm_migrations` a rename or delete of `datatable` in `w_id` should touch — +/// `None` when that is somebody else's. +/// +/// A fork's entry points at the workspace that governs the data table, so renaming or removing it +/// changes what the fork calls the data table and nothing more. Following the pointer here would +/// let a fork admin relabel or wipe the *governing* workspace's migration bookkeeping through +/// their own settings form, and the parent would then re-run every migration from zero. +async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result> { + let governing = resolve_governing_datatable(db, w_id, datatable).await?; + if governing.workspace_id != w_id { + return Ok(None); + } let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?; serde_json::from_value(db_resource) + .map(Some) .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) } @@ -1561,7 +1843,9 @@ fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> { /// Drop a data table's rows from its own database's `_wm_migrations`. async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> { - let pg_db = resolve_datatable_pg(db, w_id, datatable).await?; + let Some(pg_db) = resolve_datatable_pg(db, w_id, datatable).await? else { + return Ok(()); + }; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { let _ = connection.await; @@ -1586,7 +1870,9 @@ async fn remote_rename_datatable_migrations( from: &str, to: &str, ) -> Result<()> { - let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?; + let Some(pg_db) = resolve_datatable_pg(db, w_id, resolve_by).await? else { + return Ok(()); + }; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { let _ = connection.await; diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs new file mode 100644 index 0000000000..69e04e328d --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -0,0 +1,79 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Who may connect to a data table as which role. +//! +//! The decision lives on the data table entry of the workspace that governs it, which is not +//! necessarily the workspace asking: a fork's entry points at its parent's, and everything here +//! resolves through that pointer first. Nothing in this module runs SQL against the data table — +//! a save is tenant lists and a default, and the Postgres roles themselves are the instance +//! catalog's business. + +use axum::{routing::get, Router}; + +use windmill_api_auth::ApiAuthed; +use windmill_common::error::Result; +use windmill_common::workspaces::GoverningDatatable; +use windmill_common::DB; + +use crate::datatable_permissions_oss as roles; + +pub(crate) fn routes() -> Router { + Router::new() + .route( + "/datatable_permissions/{datatable_name}", + get(roles::get_datatable_permissions).post(roles::set_datatable_permissions), + ) + .route( + "/datatable_usable_roles/{datatable_name}", + get(roles::list_usable_datatable_roles), + ) +} + +/// Administering a data table — its permissions, its migrations that declare no role, its exports +/// — is for the admins of the workspace that governs it. A fork can use the data table; it never +/// administers it. +// The gate for whatever administers a data table under roles, which the routes of this module alone +// do not always reach. +#[allow(dead_code)] +pub(crate) async fn ensure_governs_datatable( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + governing: &GoverningDatatable, +) -> Result<()> { + roles::ensure_governs_datatable(db, authed, w_id, governing).await +} + +/// Refuse a caller that no tenant of this data table covers. +/// +/// The bookkeeping endpoints below open the data table's `admin` connection to read or create +/// `_wm_migrations` before they know which migration will run — so without this, someone covered +/// by no role at all can still force admin-backed reads and writes on a database they may not +/// touch. It asks only "may you reach this data table as anything"; which role a given migration +/// runs as is still decided per migration, and by the executor after that. +pub(crate) async fn ensure_reaches_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, +) -> Result<()> { + roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await +} + +/// [`ensure_reaches_datatable`] against an entry already resolved, for a caller that goes on to +/// connect from that same entry. +pub(crate) async fn ensure_reaches_governing_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + governing: &GoverningDatatable, + authed: &ApiAuthed, +) -> Result<()> { + roles::ensure_reaches_governing_datatable(db, w_id, datatable_name, governing, authed).await +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs new file mode 100644 index 0000000000..736493cc2d --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -0,0 +1,112 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the data table permissions endpoints and their gates come from: the enterprise +//! implementation, or a refusal. Roles are an Enterprise Edition feature; see +//! `windmill_common::datatable_roles_oss`. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_permissions_ee::{ + ensure_governs_datatable, ensure_reaches_datatable, ensure_reaches_governing_datatable, + get_datatable_permissions, list_usable_datatable_roles, set_datatable_permissions, + usable_datatable_roles, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use windmill_api_auth::ApiAuthed; + use windmill_common::{ + datatable_roles_oss::datatable_roles_unavailable as unavailable, + error::Result, + workspaces::{resolve_governing_datatable, GoverningDatatable}, + DB, + }; + + /// Nobody administers a data table's roles without them. + #[allow(dead_code)] + pub(crate) async fn ensure_governs_datatable( + _db: &DB, + _authed: &ApiAuthed, + _w_id: &str, + _governing: &GoverningDatatable, + ) -> Result<()> { + Err(unavailable()) + } + + /// A data table not under roles is reached as it was before roles existed. One under roles is + /// refused: no role of it can be connected as. + pub(crate) async fn ensure_reaches_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + _authed: &ApiAuthed, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + pub(crate) async fn ensure_reaches_governing_datatable( + _db: &DB, + _w_id: &str, + _datatable_name: &str, + governing: &GoverningDatatable, + _authed: &ApiAuthed, + ) -> Result<()> { + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + // The routes stay registered so the API has one shape; each answers after authentication, + // before anything is read. + + pub(crate) async fn get_datatable_permissions(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn set_datatable_permissions(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) struct UsableDatatableRoles { + pub(crate) permissioned: bool, + pub(crate) roles: Vec, + pub(crate) default_role: String, + } + + /// A data table not under roles is used as `admin`, as before roles existed. One under roles + /// is refused: no role of it can be connected as. + pub(crate) async fn usable_datatable_roles( + _db: &DB, + _authed: &ApiAuthed, + _w_id: &str, + governing: &GoverningDatatable, + ) -> Result { + if governing.datatable.permissions.is_some() { + return Err(unavailable()); + } + Ok(UsableDatatableRoles { + permissioned: false, + roles: vec![], + default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), + }) + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_replay_oss.rs b/backend/windmill-api-workspaces/src/datatable_replay_oss.rs new file mode 100644 index 0000000000..edcf8e402b --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_replay_oss.rs @@ -0,0 +1,48 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the replay of a data table's owners and grants into its copy comes from: the enterprise +//! one, or a refusal. Without it a data table under roles is not copied at all: its rows would +//! arrive owned by the admin connection with no grant for any role. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_replay_ee::replay_owners_and_grants; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_replay() -> windmill_common::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use { + std::collections::BTreeSet, + tokio::sync::mpsc, + tokio_postgres::error::DbError, + windmill_common::error::{Error, Result}, +}; + +/// Checked before anything is created. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn ensure_replay() -> Result<()> { + Err(Error::BadRequest( + "Cloning a data table under roles is a Windmill Enterprise Edition feature: the copy \ + needs the source's owners and grants replayed. Fork it keeping the original database \ + instead." + .to_string(), + )) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) async fn replay_owners_and_grants( + _source: &tokio_postgres::Client, + _target: &tokio_postgres::Transaction<'_>, + _notices: &mut mpsc::UnboundedReceiver, + _catalog_roles: &BTreeSet, +) -> Result<()> { + ensure_replay() +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index 017c9702a5..e7da2b8496 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,9 +1,25 @@ -pub mod datatable_migrations; -pub mod deployment_requests; +#[cfg(feature = "parquet")] +pub mod ai_session_backups; pub mod data_metrics; +pub mod datatable_acl; +pub mod datatable_acl_oss; +pub mod datatable_clone; +pub mod datatable_migrations; +pub mod datatable_permissions; +pub mod datatable_permissions_oss; +pub mod datatable_replay_oss; +pub mod deployment_requests; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; #[cfg(feature = "private")] pub mod workspaces_ee; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_acl_ee; +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_replay_ee; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_permissions_ee; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index bd51abb5c8..97d9b60de5 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -15,6 +15,7 @@ use windmill_common::email_oss::send_email_if_possible; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; +use windmill_dep_map::lock_hash::record_lock_hashes_for_workspace; use axum::{ extract::{Extension, Path, Query}, @@ -37,17 +38,19 @@ use windmill_common::global_settings::HTTP_ROUTE_WORKSPACED_ROUTE; use windmill_common::query_builders::{render_db_quoted_identifier, DbType}; use windmill_common::users::username_to_permissioned_as; use windmill_common::variables::{ - build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, + crypt_from_key_with_suffix, decrypt, encrypt, WORKSPACE_CRYPT_CACHE, }; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db_unchecked, + check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db, + get_datatable_resource_from_db_unchecked, parse_datatable_ref_for, resolve_governing_datatable, validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, - DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, - ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, + DataTableCatalogResourceType, DataTableForkBehavior, DatatableAccess, GoverningDatatable, + ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult, + WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -117,6 +120,7 @@ pub fn workspaced_service() -> Router { get(get_secondary_storage_names), ) .route("/is_premium", get(is_premium)) + .route("/billable_seats", get(get_billable_seats)) .route("/edit_error_handler", post(edit_error_handler)) .route("/edit_success_handler", post(edit_success_handler)) .route( @@ -147,6 +151,8 @@ pub fn workspaced_service() -> Router { get(test_datatable_connection), ) .merge(crate::datatable_migrations::routes()) + .merge(crate::datatable_permissions::routes()) + .merge(crate::datatable_acl::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) .route("/edit_git_sync_config", post(edit_git_sync_config)) @@ -157,6 +163,13 @@ pub fn workspaced_service() -> Router { ) .route("/edit_deploy_ui_config", post(edit_deploy_ui_config)) .route("/edit_default_app", post(edit_default_app)) + .route("/edit_guest_access", post(edit_guest_access)) + .route( + "/edit_add_admins_and_developers_to_forks", + post(edit_add_admins_and_developers_to_forks), + ) + .route("/edit_guest_jwt_key", post(edit_guest_jwt_key)) + .route("/guest_usage", get(get_guest_usage)) .route("/default_app", get(get_default_app)) .route( "/default_scripts", @@ -230,6 +243,10 @@ pub fn global_service() -> Router { .route("/list", get(list_workspaces)) .route("/users", get(user_workspaces)) .route("/session_workspace_status", post(session_workspace_status)) + .route( + "/session_workspace_retention", + post(session_workspace_retention), + ) .route("/create", post(create_workspace)) .route("/create_fork", post(deprecated_create_workspace_fork)) .route("/exists", post(exists_workspace)) @@ -325,6 +342,18 @@ pub struct WorkspaceSettings { #[serde(skip_serializing_if = "Option::is_none")] pub public_app_execution_limit_per_minute: Option, pub error_handler_fallback_to_instance_alerts: bool, + /// Whether this workspace admits guest sessions (`ExecutionMode::Guest`). An app's + /// own `execution_mode: guest` is inert while this is off. + pub guest_access_enabled: bool, + /// The key a guest JWT is verified against: a PEM public key, or a JWKS URL, at most + /// one (a DB CHECK enforces it). Public material, not a secret, so it is admin- + /// readable here. `None`/`None` falls back to the instance issuer (`JWT_EXT_JWKS_URL`) + /// off cloud, or accepts no JWT guest if none is set; `guest_access_enabled` is the switch. + #[serde(skip_serializing_if = "Option::is_none")] + pub guest_jwt_public_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub guest_jwt_jwks_url: Option, + pub add_admins_and_developers_to_forks: bool, } /// Subset of `WorkspaceSettings` that is safe to return to any workspace @@ -347,6 +376,11 @@ pub struct WorkspacePublicSettings { pub teams_team_guid: Option, #[serde(skip_serializing_if = "Option::is_none")] pub mute_critical_alerts: Option, + /// Not sensitive, and the app editor needs it to say whether the guest rung is + /// live -- an app can be set to `guest` while the workspace has guests off. + pub guest_access_enabled: bool, + /// Read by the fork dialog, which tells the forker who else the fork will include. + pub add_admins_and_developers_to_forks: bool, #[serde(skip_serializing_if = "Option::is_none")] pub deploy_ui: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -484,8 +518,8 @@ struct CreateWorkspaceFork { id: String, name: String, color: Option, - /// Datatable names that were forked. For each, the backend will update the - /// forked workspace's datatable config to point to the new database. + /// Data tables the fork gets a copy of rather than a pointer at the parent's. For each, the + /// fork's entry is pointed at the new database. #[serde(default)] forked_datatables: Vec, /// Lakes the user explicitly chose to SHARE with the parent (the fork then reads and @@ -517,6 +551,11 @@ struct CreateWorkspaceFork { struct ForkedDatatableInfo { name: String, new_dbname: String, + /// What this request copies into `new_dbname`, which it creates. Optional only so that the + /// entry older CLIs send — naming a database they created and filled themselves — is refused + /// with a message rather than a parse error. + #[serde(default)] + fork_behavior: Option, } #[derive(Deserialize)] @@ -546,6 +585,7 @@ struct UserWorkspace { /// screen off this. pub created_by: Option, pub disabled: bool, + pub is_service_account: bool, } #[derive(Deserialize)] @@ -703,6 +743,48 @@ async fn is_premium( Ok(Json(premium)) } +#[derive(Serialize)] +struct BillableSeatsResponse { + /// Both omitted when the seats counted are another workspace's: a fork member need not be a + /// member of the billing root, so the root's headcount is not theirs to read. The total is, + /// since it is the divisor of the quota their own executions draw on. + #[serde(skip_serializing_if = "Option::is_none")] + developers: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operators: Option, + seats: i64, +} + +async fn get_billable_seats( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + // Readable by any workspace member, like `is_premium`: this is what the sidebar usage meter + // divides by, and that meter is shown to non-admin developers too. + // + // On cloud a fork draws its plan, quota and bill from the root, so the seats its usage is + // measured against are the root's. Resolved here rather than by the caller: a fork member need + // not be a member of that root, and so cannot count its seats from the member list. Off cloud + // a fork is not billed through a root at all, so the workspace answers for itself. + #[cfg(feature = "cloud")] + let billing_w_id = if *CLOUD_HOSTED { + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id).await? + } else { + w_id.clone() + }; + #[cfg(not(feature = "cloud"))] + let billing_w_id = w_id.clone(); + + let counted = windmill_common::workspaces::billable_seats(&db, &billing_w_id).await?; + let own = billing_w_id == w_id; + Ok(Json(BillableSeatsResponse { + developers: own.then_some(counted.developers), + operators: own.then_some(counted.operators), + seats: counted.seats, + })) +} + async fn exists_workspace( authed: ApiAuthed, Extension(user_db): Extension, @@ -866,12 +948,15 @@ fn redact_git_sync_webhook_secrets(git_sync: &mut serde_json::Value) { } /// Zero the server-owned auto-pull fields (webhook id/secret/url/error, synced -/// sha, last pull status) on a client-supplied `AutoPullSettings`. The client only -/// controls `enabled` / `mode` / `poll_interval_s`; the rest is written by the -/// server (webhook creation, poller) and must never be trusted from the request — -/// otherwise a caller could inject a webhook id/secret or fake sync state. -fn clear_client_supplied_auto_pull_state( +/// sha, last pull status) on a client-supplied `AutoPullSettings`, and stamp who +/// pulls run as: `saver_email` while auto pull is on. The client only controls +/// `enabled` / `mode` / `poll_interval_s`; the rest is written by the server (webhook +/// creation, poller, this save) and must never be trusted from the request — +/// otherwise a caller could inject a webhook id/secret, fake sync state, or pick who +/// pulls run as. +fn sanitize_client_auto_pull( auto_pull: &mut windmill_common::workspaces::AutoPullSettings, + saver_email: &str, ) { auto_pull.webhook_id = None; auto_pull.webhook_secret = None; @@ -879,6 +964,28 @@ fn clear_client_supplied_auto_pull_state( auto_pull.webhook_error = None; auto_pull.last_synced_sha = std::collections::HashMap::new(); auto_pull.last_pull_status = None; + auto_pull.enabled_by = auto_pull.enabled.then(|| saver_email.to_string()); +} + +#[cfg(test)] +mod sanitize_client_auto_pull_tests { + use windmill_common::workspaces::AutoPullSettings; + + #[test] + fn a_save_stamps_the_saver_over_any_client_supplied_stamp() { + let mut ap = AutoPullSettings { + enabled: true, + enabled_by: Some("forged@example.com".to_string()), + ..Default::default() + }; + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by.as_deref(), Some("saver@example.com")); + + ap.enabled = false; + ap.enabled_by = Some("forged@example.com".to_string()); + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by, None, "auto pull off carries no stamp"); + } } /// Whether a git-sync repository tracking `tracked` rules out `label_branch` as a dev workspace's @@ -1047,7 +1154,11 @@ async fn get_settings( success_handler, variable_expiration_handler, public_app_execution_limit_per_minute, - error_handler_fallback_to_instance_alerts + error_handler_fallback_to_instance_alerts, + guest_access_enabled, + guest_jwt_public_key, + guest_jwt_jwks_url, + add_admins_and_developers_to_forks FROM workspace_settings WHERE @@ -1065,6 +1176,8 @@ async fn get_settings( if let Some(git_sync) = settings.git_sync.as_mut() { redact_git_sync_webhook_secrets(git_sync); } + settings.datatable = + windmill_common::workspaces::strip_datatable_permissions(settings.datatable.take()); Ok(Json(settings)) } @@ -1086,6 +1199,8 @@ async fn get_public_settings( teams_team_name, teams_team_guid, mute_critical_alerts, + guest_access_enabled, + add_admins_and_developers_to_forks, deploy_ui, large_file_storage, datatable @@ -1100,12 +1215,26 @@ async fn get_public_settings( .await .map_err(|e| Error::internal_err(format!("getting public settings: {e:#}")))?; - let settings = not_found_if_none(settings, "workspace settings", &w_id)?; + let mut settings = not_found_if_none(settings, "workspace settings", &w_id)?; tx.commit().await?; + settings.datatable = + windmill_common::workspaces::strip_datatable_permissions(settings.datatable.take()); Ok(Json(settings)) } +/// The instance's standing against the guest allowance: counts only, no emails, so any +/// member may read it. Instance-wide, since a licence is per instance and one email is +/// one guest however many workspaces it opens; the settings card and the editor's +/// Guests rung show it so nobody discovers the cap from a visitor's complaint. +async fn get_guest_usage( + _authed: ApiAuthed, + Extension(db): Extension, + Path(_w_id): Path, +) -> JsonResult { + Ok(Json(windmill_common::workspaces::guest_usage(&db).await?)) +} + #[derive(Deserialize)] pub struct GitSyncDeployModeQuery { /// The branch the caller would push. @@ -1220,26 +1349,22 @@ async fn get_git_sync_deploy_mode( let configured = !settings.repositories.is_empty(); - // Auto-pull runs only on Enterprise-licensed instances (see poll_git_auto_pull); - // without a caller branch there is nothing to match. Either way deploy_on_push - // stays false and the caller falls back (git push via CI, or wmill sync push). + // Auto-pull runs only in builds that compile the poller (`private`); without a + // caller branch there is nothing to match. Either way deploy_on_push stays + // false and the caller falls back (git push via CI, or wmill sync push). let Some(branch) = q.branch.as_deref() else { return Ok(Json(GitSyncDeployMode { configured, deploy_on_push: false, })); }; - let licensed = matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ); // Count the auto-pull repos that would deploy this branch. We deliberately do // not check the caller's remote URL: with exactly one such repo the local // checkout is unambiguously it, and with several we can't tell which is the // caller's, so we report false and let the CLI ask the user. let mut matches = 0u32; - if licensed && !root_deleted { + if cfg!(feature = "private") && !root_deleted { for repo in &settings.repositories { let Some(auto_pull) = repo.auto_pull.as_ref() else { continue; @@ -2052,6 +2177,17 @@ async fn edit_large_file_storage_config( serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; + // A workspace whose AI session backups fell back to the instance store leaves it + // here: the generation moves on, so nothing it left in any instance store is read + // again, whichever one a later return to the fallback finds (`ai_session_backups`). + sqlx::query!( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 \ + WHERE workspace_id = $1 AND large_file_storage IS NULL", + &w_id + ) + .execute(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", serialized_lfs_config, @@ -2067,8 +2203,23 @@ async fn edit_large_file_storage_config( .execute(&mut *tx) .await?; } + let backups_generation = sqlx::query_scalar!( + "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; + // Read by nothing any more, whatever the storage is now: what the AI session backups + // left in the instance store under a generation older than the one just committed. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_fallback(w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Trigger git sync for large file storage changes handle_deployment_metadata( &authed.email, @@ -2117,6 +2268,12 @@ struct DataTableListItem { name: String, resource_type: String, resource_path: String, + /// The workspace whose entry governs this one, when it is not this workspace — a fork pointing + /// at its parent. Its permissions apply here, and only its admins may edit them. + #[serde(skip_serializing_if = "Option::is_none")] + governing_workspace_id: Option, + /// Whether the governing entry is under roles. + permissioned: bool, } async fn list_datatables( @@ -2124,26 +2281,29 @@ async fn list_datatables( Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { - let config = sqlx::query_scalar!( - "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // A pointer entry owns no database, so what it resolves to is the only truthful answer here. + // One that resolves to nothing — a pointer whose workspace was deleted — is dropped rather than + // listed with a database it does not have; what happened is named where it is actionable + // instead: by the delete that stranded it, and by any attempt to use it. + let resolved = + windmill_common::workspaces::resolve_workspace_governing_datatables(&db, &w_id).await?; - let items: Vec = match config { - Some(val) => { - let map: HashMap = serde_json::from_value(val).unwrap_or_default(); - map.into_iter() - .map(|(name, dt)| DataTableListItem { - name, - resource_type: dt.database.resource_type.as_ref().to_string(), - resource_path: dt.database.resource_path, - }) - .collect() - } - None => vec![], - }; + let mut items = Vec::with_capacity(resolved.len()); + for (name, governing) in resolved { + let database = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database"); + items.push(DataTableListItem { + name, + resource_type: database.resource_type.as_ref().to_string(), + resource_path: database.resource_path.clone(), + governing_workspace_id: (governing.governing_workspace_id() != w_id) + .then(|| governing.governing_workspace_id().to_string()), + permissioned: governing.datatable.permissions.is_some(), + }); + } Ok(Json(items)) } @@ -2179,6 +2339,25 @@ struct DataTableTables { schemas: TableListMap, #[serde(skip_serializing_if = "Option::is_none")] error: Option, + /// On the instance database: the only kind that can be under roles or have its access edited. + instance: bool, + permissioned: bool, + /// The roles this caller may connect as, by name; empty when not under roles. + usable_roles: Vec, + default_role: String, + /// What the role the listing connected as may create. + can_create_schema: bool, + creatable_schemas: Vec, +} + +#[derive(Deserialize)] +struct ListDataTableTablesQuery { + /// List only this data table: each entry opens a connection to its database. + datatable_name: Option, + /// The data table `role` applies to. Every other one is listed as its default role, since a + /// role name means nothing outside the data table it belongs to. + role_for: Option, + role: Option, } #[derive(Deserialize)] @@ -2186,6 +2365,7 @@ struct GetDataTableSchemaQuery { datatable_name: String, schema_name: String, table_name: String, + role: Option, } #[derive(Serialize, Debug)] @@ -2231,6 +2411,15 @@ async fn test_datatable_connection( Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { require_admin(authed.is_admin, &authed.username)?; + // Reports what the admin connection can do, so it answers to the workspace that governs the + // data table rather than to whichever one is asking. + windmill_common::workspaces::ensure_datatable_admin_access( + &db, + &w_id, + &datatable_name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?; let pg_db: PgDatabase = serde_json::from_value(db_resource) @@ -2318,7 +2507,7 @@ async fn test_datatable_connection( } async fn list_datatable_schemas( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { @@ -2326,7 +2515,7 @@ async fn list_datatable_schemas( let mut results = Vec::new(); for datatable_name in datatable_names { - let schema = match get_datatable_schema(&db, &w_id, &datatable_name).await { + let schema = match get_datatable_schema(&db, &authed, &w_id, &datatable_name).await { Ok(schemas) => DataTableSchema { datatable_name, schemas, error: None }, Err(e) => DataTableSchema { datatable_name, @@ -2341,40 +2530,106 @@ async fn list_datatable_schemas( } async fn list_datatable_tables( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, + Query(query): Query, ) -> JsonResult> { - let datatable_names = list_datatable_names(&db, &w_id).await?; + if query.role.is_some() && query.role_for.is_none() { + return Err(Error::BadRequest( + "`role` needs `role_for`, the data table it is a role of".to_string(), + )); + } + if let (Some(only), Some(role_for)) = + (query.datatable_name.as_deref(), query.role_for.as_deref()) + { + if only != role_for { + return Err(Error::BadRequest(format!( + "`role_for` names '{role_for}', which `datatable_name` leaves out of the listing" + ))); + } + } + let mut datatable_names = list_datatable_names(&db, &w_id).await?; + for named in [query.role_for.as_deref(), query.datatable_name.as_deref()] + .into_iter() + .flatten() + { + if !datatable_names.iter().any(|n| n == named) { + return Err(Error::NotFound(format!( + "No data table named '{named}' in this workspace" + ))); + } + } + if let Some(only) = query.datatable_name.as_deref() { + datatable_names.retain(|n| n == only); + } let mut results = Vec::new(); for datatable_name in datatable_names { - let tables = match get_datatable_tables(&db, &w_id, &datatable_name).await { - Ok(schemas) => DataTableTables { datatable_name, schemas, error: None }, - Err(e) => DataTableTables { - datatable_name, - schemas: HashMap::new(), - error: Some(e.to_string()), - }, - }; - results.push(tables); + let role = query + .role + .as_deref() + .filter(|_| query.role_for.as_deref() == Some(datatable_name.as_str())); + results.push(list_one_datatable_tables(&db, &authed, &w_id, datatable_name, role).await); } Ok(Json(results)) } +async fn list_one_datatable_tables( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: String, + role: Option<&str>, +) -> DataTableTables { + let mut entry = DataTableTables { + datatable_name, + schemas: HashMap::new(), + error: None, + instance: false, + permissioned: false, + usable_roles: vec![], + default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), + can_create_schema: false, + creatable_schemas: vec![], + }; + let result: Result<()> = async { + let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?; + entry.instance = governing.is_instance(); + let usable = + crate::datatable_permissions_oss::usable_datatable_roles(db, authed, w_id, &governing) + .await?; + entry.permissioned = usable.permissioned; + entry.usable_roles = usable.roles; + entry.default_role = usable.default_role; + let listing = get_datatable_tables(db, authed, w_id, &entry.datatable_name, role).await?; + entry.schemas = listing.schemas; + entry.can_create_schema = listing.can_create_schema; + entry.creatable_schemas = listing.creatable_schemas; + Ok(()) + } + .await; + if let Err(e) = result { + entry.error = Some(e.to_string()); + } + entry +} + async fn get_datatable_table_schema( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, Query(query): Query, ) -> JsonResult { let columns = get_datatable_table_columns( &db, + &authed, &w_id, &query.datatable_name, &query.schema_name, &query.table_name, + query.role.as_deref(), ) .await?; @@ -2402,13 +2657,38 @@ async fn list_datatable_names(db: &DB, w_id: &str) -> Result> { .collect()) } -async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Result { - // Get the datatable resource (connection credentials) - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; +/// Connect to a data table as the caller, not as `admin`: the role they named, or the data table's +/// default. A data table not under roles resolves as `admin`, exactly as it did before roles. +/// +/// Every schema-browsing query below then reports what this Postgres role can actually reach, +/// which is why they filter on `has_schema_privilege` — `pg_catalog` is world-readable, so an +/// unfiltered listing would name schemas the connection cannot even enter. +async fn resolve_datatable_pg_as_caller( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, + role: Option<&str>, +) -> Result { + let db_resource = get_datatable_resource_from_db( + db, + w_id, + datatable_name, + role, + DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} - // Parse the resource as PgDatabase - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; +async fn get_datatable_schema( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, None).await?; // Connect to the datatable database let (client, connection) = pg_db.connect(Some(db)).await?; @@ -2428,6 +2708,7 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu FROM pg_namespace WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') AND nspname NOT LIKE 'pg_%' + AND has_schema_privilege(oid, 'USAGE') ORDER BY nspname "#, &[], @@ -2495,10 +2776,20 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu Ok(schema_map) } -async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Result { - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; +struct DatatableTableListing { + schemas: TableListMap, + can_create_schema: bool, + creatable_schemas: Vec, +} + +async fn get_datatable_tables( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, + role: Option<&str>, +) -> Result { + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2510,10 +2801,11 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu let schema_rows = client .query( r#" - SELECT nspname::text AS schema_name + SELECT nspname::text AS schema_name, has_schema_privilege(oid, 'CREATE') AS can_create FROM pg_namespace WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') AND nspname NOT LIKE 'pg_%' + AND has_schema_privilege(oid, 'USAGE') ORDER BY nspname "#, &[], @@ -2523,11 +2815,29 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu Error::internal_err(format!("Failed to query schemas: {}", pg_error_message(&e))) })?; + let can_create_schema: bool = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CREATE')", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read database privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + let mut table_map: TableListMap = HashMap::new(); + let mut creatable_schemas = Vec::new(); let schema_names: Vec = schema_rows .iter() .map(|row| { let name: String = row.get(0); + if row.get::<_, bool>(1) { + creatable_schemas.push(name.clone()); + } table_map.entry(name.clone()).or_default(); name }) @@ -2557,15 +2867,17 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu table_map.entry(table_schema).or_default().push(table_name); } - Ok(table_map) + Ok(DatatableTableListing { schemas: table_map, can_create_schema, creatable_schemas }) } async fn get_datatable_table_columns( db: &DB, + authed: &ApiAuthed, w_id: &str, datatable_name: &str, schema_name: &str, table_name: &str, + role: Option<&str>, ) -> Result { if is_system_pg_schema(schema_name) { return Err(Error::BadRequest(format!( @@ -2574,9 +2886,7 @@ async fn get_datatable_table_columns( ))); } - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2672,6 +2982,78 @@ fn truncate_column_default(default: String) -> String { mod tests { use super::*; + #[test] + fn a_dev_workspace_copies_into_the_fork_database_namespace() { + assert_eq!( + forked_datatable_dbname("wm-fork-my-fork", "main"), + "wm_fork_my_fork__main" + ); + assert_eq!( + forked_datatable_dbname("my-dev", "main"), + "wm_fork_my_dev__main" + ); + } + + /// The header of a pg_dump, followed by an object whose body also holds a `SET`. + const DUMP: &str = "--\n\ + -- PostgreSQL database dump\n\ + --\n\ + \n\ + \\restrict aBcD\n\ + \n\ + SET statement_timeout = 0;\n\ + SET transaction_timeout = 0;\n\ + SET client_encoding = 'UTF8';\n\ + SELECT pg_catalog.set_config('search_path', '', false);\n\ + \n\ + SET default_table_access_method = heap;\n\ + \n\ + CREATE FUNCTION public.f() RETURNS void LANGUAGE plpgsql AS $$\n\ + BEGIN\n\ + SET transaction_timeout = 0;\n\ + END;\n\ + $$;\n"; + + #[test] + fn replayable_dump_keeps_everything_but_meta_commands_and_session_timeouts() { + let replayable = strip_unreplayable_dump_lines(DUMP); + + assert!(!replayable.contains("\\restrict")); + assert!(!replayable.contains("SET statement_timeout")); + assert!(!replayable.contains("SET transaction_timeout = 0;\nSET client_encoding")); + assert!(replayable.contains("SET client_encoding = 'UTF8';")); + assert!(replayable.contains("SET default_table_access_method = heap;")); + // Past the preamble the dump is an object's own text: left exactly as it is. + assert!(replayable.contains("BEGIN\nSET transaction_timeout = 0;\nEND;")); + } + + #[tokio::test] + async fn dump_preamble_only_drops_settings_the_server_lacks() { + let dump_file = DumpFile::new().unwrap(); + tokio::fs::write(&dump_file.path, DUMP).await.unwrap(); + let supported = [ + "statement_timeout", + "client_encoding", + "default_table_access_method", + ] + .map(String::from) + .into_iter() + .collect(); + + comment_out_unsupported_settings(&dump_file, &supported) + .await + .unwrap(); + + let patched = tokio::fs::read_to_string(&dump_file.path).await.unwrap(); + // Rewriting the header must not shift the rest of the dump. + assert_eq!(patched.len(), DUMP.len()); + assert!(patched.contains("-- transaction_timeout = 0;")); + assert!(patched.contains("SET statement_timeout = 0;")); + assert!(patched.contains("SET default_table_access_method = heap;")); + // The `SET` inside the function body is past the preamble: never touched. + assert!(patched.contains("BEGIN\nSET transaction_timeout = 0;\nEND;")); + } + #[test] fn compact_column_type_truncates_multibyte_defaults_safely() { let default = "é".repeat(31); @@ -2726,7 +3108,10 @@ mod tests { } /// Resolve a source string to PgDatabase credentials with user-scoped permission checks. -/// For `datatable://name`: accessible to everyone (variables are resolved internally). +/// +/// For `datatable://name`: the **admin** connection, so it is gated on admin reach. Every caller +/// copies, dumps or drops a whole database, and a dump taken under a restricted role would be a +/// silently truncated copy rather than an error — which is worse than refusing. /// For `$res:path`: uses UserDB (row-level security) to verify the user can see the resource, /// then interpolates `$var:` references in the resource value. pub(crate) async fn resolve_pg_source_checked( @@ -2737,6 +3122,13 @@ pub(crate) async fn resolve_pg_source_checked( source: &str, ) -> Result { let db_resource = if let Some(name) = source.strip_prefix("datatable://") { + windmill_common::workspaces::ensure_datatable_admin_access( + db, + w_id, + name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; get_datatable_resource_from_db_unchecked(db, w_id, name).await? } else if let Some(path) = source.strip_prefix("$res:") { let db_with_authed = windmill_common::db::DbWithOptAuthed::from_authed( @@ -2774,6 +3166,26 @@ pub(crate) async fn resolve_pg_source_checked( .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) } +/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL +/// rather than a user resource. +pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result { + // Resolved rather than read: a pointer entry owns no database of its own, so only the entry it + // lands on can answer. A name that resolves to nothing keeps the historical `false`. + Ok(resolve_governing_datatable(db, w_id, name) + .await + .ok() + .and_then(|g| g.datatable.database) + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)) +} + +/// Same, for the `datatable://` / `$res:` form the import endpoints take. +async fn is_instance_datatable_source(db: &DB, w_id: &str, source: &str) -> Result { + match source.strip_prefix("datatable://") { + Some(name) => is_instance_datatable(db, w_id, name).await, + None => Ok(false), + } +} + /// A temporary file for pg_dump output that is automatically deleted when dropped. pub(crate) struct DumpFile { pub(crate) path: std::path::PathBuf, @@ -2821,12 +3233,21 @@ impl Drop for DumpFile { } } +#[derive(Default)] +pub(crate) struct PgDumpOptions<'a> { + pub(crate) schema_only: bool, + pub(crate) exclude_tables: &'a [&'a str], + /// Leave out `ALTER ... OWNER TO`. + pub(crate) no_owner: bool, + /// Leave out `GRANT`, `REVOKE` and `ALTER DEFAULT PRIVILEGES`. + pub(crate) no_acl: bool, +} + /// Run pg_dump against a PgDatabase, writing output to a temp file on disk. /// Returns a DumpFile handle; the file is deleted when the handle is dropped. pub(crate) async fn pg_dump_database( pg_db: &PgDatabase, - schema_only: bool, - exclude_tables: &[&str], + opts: PgDumpOptions<'_>, ) -> Result { let dump_file = DumpFile::new()?; @@ -2837,10 +3258,16 @@ pub(crate) async fn pg_dump_database( let mut cmd = tokio::process::Command::new("pg_dump"); cmd.arg("--format=plain").arg("--file").arg(&dump_file.path); - if schema_only { + if opts.schema_only { cmd.arg("--schema-only"); } - for table in exclude_tables { + if opts.no_owner { + cmd.arg("--no-owner"); + } + if opts.no_acl { + cmd.arg("--no-privileges"); + } + for table in opts.exclude_tables { cmd.arg(format!("--exclude-table={table}")); } cmd.arg("--host") @@ -2872,37 +3299,182 @@ pub(crate) async fn pg_dump_database( Ok(dump_file) } -/// Import a pg_dump file into a target database using psql. -async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { - let host = &target_db.host; - let port = target_db.port.unwrap_or(5432).to_string(); - let user = target_db.login_name(); - let dbname = &target_db.dbname; +/// Whether `line` still belongs to the preamble pg_dump emits before the first +/// dumped object: comments, blank lines, psql meta-commands and the session `SET`s. +fn is_dump_preamble_line(line: &[u8]) -> bool { + let line = line.trim_ascii_start(); + line.is_empty() + || line.starts_with(b"--") + || line.starts_with(b"\\") + || line.starts_with(b"SET ") + || line.starts_with(b"SELECT pg_catalog.set_config(") +} +/// The GUCs pg_dump's preamble sets only to keep the dumping session out of the way. +/// They are also the ones that come and go across versions (`transaction_timeout` is +/// PG 17+), so they are what a dump replayed on an older server trips over first. +const DUMP_SESSION_TIMEOUTS: [&str; 4] = [ + "statement_timeout", + "lock_timeout", + "idle_in_transaction_session_timeout", + "transaction_timeout", +]; + +/// Turn a dump into SQL that can be replayed on another database: drop pg_dump's psql +/// meta-commands (`\restrict` / `\unrestrict`, not valid SQL) and the session timeouts +/// its preamble sets, which the replaying server may not have as GUCs at all. Only the +/// preamble is filtered, so an object's body keeps whatever it holds. +pub(crate) fn strip_unreplayable_dump_lines(dump: &str) -> String { + let mut in_preamble = true; + dump.lines() + .filter(|line| { + in_preamble = in_preamble && is_dump_preamble_line(line.as_bytes()); + if line.trim_start().starts_with('\\') { + return false; + } + !(in_preamble + && preamble_setting_name(line.as_bytes()) + .is_some_and(|name| DUMP_SESSION_TIMEOUTS.contains(&name))) + }) + .collect::>() + .join("\n") +} + +/// The GUC a preamble `SET = ...;` line assigns, if the line is one. +fn preamble_setting_name(line: &[u8]) -> Option<&str> { + let name = line.strip_prefix(b"SET ")?.split(|c| *c == b' ').next()?; + std::str::from_utf8(name).ok() +} + +/// The preamble Windmill's postgres client writes can set GUCs an older server does not +/// have — harmless session tuning, but one failing statement aborts a restore that stops +/// on the first error. Comment those out in place, three bytes each, so the data +/// section's offsets stay put. +async fn comment_out_unsupported_settings( + dump_file: &DumpFile, + supported_settings: &HashSet, +) -> Result<()> { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + + let file = tokio::fs::File::open(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to open dump file: {}", e)))?; + let mut reader = tokio::io::BufReader::new(file); + + let mut preamble: Vec = Vec::new(); + let mut patched = false; + loop { + let start = preamble.len(); + let read = reader + .read_until(b'\n', &mut preamble) + .await + .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e)))?; + if read == 0 { + break; + } + let line = &preamble[start..]; + if !is_dump_preamble_line(line) { + preamble.truncate(start); + break; + } + if preamble_setting_name(line).is_some_and(|name| !supported_settings.contains(name)) { + preamble[start..start + 3].copy_from_slice(b"-- "); + patched = true; + } + } + if !patched { + return Ok(()); + } + + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .open(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to open dump file: {}", e)))?; + file.write_all(&preamble) + .await + .map_err(|e| Error::internal_err(format!("Failed to rewrite dump preamble: {}", e)))?; + file.flush() + .await + .map_err(|e| Error::internal_err(format!("Failed to rewrite dump preamble: {}", e)))?; + Ok(()) +} + +/// A psql invocation against `pg_db`, carrying the connection settings the CLI reads +/// from the environment. +fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { let mut cmd = tokio::process::Command::new("psql"); cmd.arg("--host") - .arg(host) + .arg(&pg_db.host) .arg("--port") - .arg(&port) + .arg(pg_db.port.unwrap_or(5432).to_string()) .arg("--username") - .arg(user) + .arg(pg_db.login_name()) .arg("--dbname") - .arg(dbname) + .arg(&pg_db.dbname) .arg("--no-psqlrc") - .arg("--file") - .arg(&dump_file.path) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); - if let Some(ref password) = target_db.password { + if let Some(ref password) = pg_db.password { cmd.env("PGPASSWORD", password); } - - if let Some(ref sslmode) = target_db.sslmode { + if let Some(ref sslmode) = pg_db.sslmode { cmd.env("PGSSLMODE", sslmode); } + cmd +} - let output = cmd +/// GUC names the server backing `pg_db` knows about. +/// +/// Asked through psql rather than a tokio-postgres connection so the lookup reaches +/// exactly the servers the restore itself can: libpq negotiates TLS for `sslmode=prefer` +/// and an unset mode, where `PgDatabase::connect` would hand a TLS-only server a +/// plaintext socket and fail before the import ever starts. +async fn server_setting_names(pg_db: &PgDatabase) -> Result> { + let output = psql_command(pg_db) + .arg("--tuples-only") + .arg("--no-align") + .arg("--command") + .arg("SELECT name FROM pg_settings") + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!( + "Failed to list the settings of the target server: {}", + stderr + ))); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect()) +} + +/// Import a pg_dump file into a target database using psql. +/// +/// Left to its defaults psql reports a failed statement, carries on and still exits 0, +/// so a dump that breaks partway through imports partially and reads as a success. +/// ON_ERROR_STOP surfaces the failure and --single-transaction makes the restore +/// all-or-nothing, leaving the target as it was and the import retryable. +/// +/// Authorization: none. Writes into `target_db` with the credentials it carries; callers MUST have +/// authorized writing to that database. +pub(crate) async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { + let supported_settings = server_setting_names(target_db).await?; + comment_out_unsupported_settings(dump_file, &supported_settings).await?; + + let output = psql_command(target_db) + .arg("--set") + .arg("ON_ERROR_STOP=1") + .arg("--single-transaction") + .arg("--file") + .arg(&dump_file.path) .output() .await .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; @@ -2937,6 +3509,14 @@ async fn create_pg_database( ) -> Result { windmill_common::validate_dbname(&req.target_dbname)?; + // The copy this database is for is refused a call later, and nothing collects an instance + // database that no data table entry names. Refuse here too, so the clone stops before one + // exists rather than leaving an empty registered `wm_fork_…` behind. + if let Some(reference) = req.source.strip_prefix("datatable://") { + let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; + ensure_copied_without_roles(&ensure_datatable_is_clonable(&db, &w_id, &name).await?)?; + } + // Non-superadmin: restrict dbname to wm_fork_ prefix if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? { if !req.target_dbname.starts_with("wm_fork_") { @@ -2947,78 +3527,68 @@ async fn create_pg_database( } } - // Determine if this is an instance or resource-backed datatable - let is_instance_datatable = if let Some(dt_name) = req.source.strip_prefix("datatable://") { - let config = sqlx::query_scalar!( - "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", - &w_id, - dt_name - ) - .fetch_optional(&db) - .await? - .flatten(); - config - .and_then(|v| { - v.get("database") - .and_then(|d| d.get("resource_type")) - .and_then(|r| r.as_str()) - .map(|s| s == "instance") - }) - .unwrap_or(false) - } else { - false - }; - - if is_instance_datatable { + if is_instance_datatable_source(&db, &w_id, &req.source).await? { windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") .await?; } else { let source_pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let (client, connection) = source_pg.connect(Some(&db)).await?; - let join_handle = tokio::spawn(async move { connection.await }); - - let row = client - .query_one( - "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", - &[&req.target_dbname], - ) - .await - .map_err(|e| { - Error::internal_err(format!( - "Failed to check database existence: {}", - pg_error_message(&e) - )) - })?; - let db_exists: bool = row.get(0); - - if db_exists { - drop(client); - let _ = windmill_common::shutdown_pg_connection(join_handle).await; - return Err(Error::BadRequest(format!( - "Database '{}' already exists on the resource server", - req.target_dbname - ))); - } - - client - .execute(&format!("CREATE DATABASE \"{}\"", &req.target_dbname), &[]) - .await - .map_err(|e| { - Error::internal_err(format!( - "Failed to create database '{}': {}", - req.target_dbname, - pg_error_message(&e) - )) - })?; - - drop(client); - windmill_common::shutdown_pg_connection(join_handle).await?; + create_database_on_server(&db, &source_pg, &req.target_dbname).await?; } Ok(format!("Created database '{}'", req.target_dbname)) } +/// `CREATE DATABASE` on the server `server` connects to, refusing a name already taken there. +/// +/// Authorization: none. Callers MUST have authorized creating a database on that server — resolved +/// from a data table or resource the caller may administer. +pub(crate) async fn create_database_on_server( + db: &DB, + server: &PgDatabase, + dbname: &str, +) -> Result<()> { + windmill_common::validate_dbname(dbname)?; + let (client, connection) = server.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let row = client + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + &[&dbname], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to check database existence: {}", + pg_error_message(&e) + )) + })?; + let db_exists: bool = row.get(0); + + if db_exists { + drop(client); + let _ = windmill_common::shutdown_pg_connection(join_handle).await; + return Err(Error::BadRequest(format!( + "Database '{dbname}' already exists on the resource server" + ))); + } + + client + .execute(&format!("CREATE DATABASE \"{dbname}\""), &[]) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to create database '{dbname}': {}", + pg_error_message(&e) + )) + })?; + + drop(client); + windmill_common::shutdown_pg_connection(join_handle).await?; + Ok(()) +} + #[derive(Deserialize)] struct ImportPgDatabaseRequest { source: String, @@ -3028,6 +3598,56 @@ struct ImportPgDatabaseRequest { fork_behavior: DataTableForkBehavior, } +/// Whether data table `name` of `w_id` has a shape a copy can be made of, returning what governs +/// it. Checked before anything is created — by the fork request for the copies it makes, and by +/// `create_pg_database` — and again when the fork's entry is written. +/// +/// It is not the check for a data table under roles: a copy of one is only correct with its +/// owners and grants replayed, which the fork request does and the older endpoints refuse +/// ([`ensure_copied_without_roles`]). +/// +/// Authorization: none. Callers MUST have authorized reaching data table `name` of `w_id` before +/// using what it returns. +pub(crate) async fn ensure_datatable_is_clonable( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + // The copy has to name a database of its own. A resource-backed entry reached through a + // pointer names one this workspace does not own, so there is nothing here to repoint. + let is_instance = governing + .datatable + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance); + if governing.workspace_id != w_id && !is_instance { + return Err(Error::BadRequest(format!( + "Data table '{name}' points at a resource-backed data table in another workspace \ + and cannot be copied; fork it from the workspace that owns it." + ))); + } + Ok(governing) +} + +/// Refuse a copy of a data table under roles through the endpoints that copy rows and nothing +/// else. +/// +/// `pg_dump` carries no roles and the import runs with `--no-privileges`, so such a copy arrives +/// with its objects owned by the admin connection and no `GRANT` for any role: the tenants pass +/// Windmill's check, connect as the role they were given, and are denied by Postgres on everything. +/// A fork request that makes the copy itself replays the owners and grants instead. +fn ensure_copied_without_roles(governing: &GoverningDatatable) -> Result<()> { + if governing.datatable.permissions.is_some() { + return Err(Error::BadRequest(format!( + "Data table '{}' is under roles, so a copy has to carry its owners and grants: \ + clone it through the fork wizard or `wmill workspace fork`, which do.", + governing.name + ))); + } + Ok(()) +} + /// Import (pg_dump/pg_import) from source to target async fn import_pg_database( authed: ApiAuthed, @@ -3040,6 +3660,11 @@ async fn import_pg_database( return Ok("No action needed for KeepOriginal behavior".to_string()); } + if let Some(reference) = req.source.strip_prefix("datatable://") { + let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; + ensure_copied_without_roles(&ensure_datatable_is_clonable(&db, &w_id, &name).await?)?; + } + if req.fork_behavior == DataTableForkBehavior::SchemaAndData { require_admin(authed.is_admin, &authed.username)?; if *CLOUD_HOSTED { @@ -3067,7 +3692,18 @@ async fn import_pg_database( } windmill_common::validate_dbname(&target_pg.dbname)?; - let dump_file = pg_dump_database(&source_pg, schema_only, &[]).await?; + // Ownership never replays: the restore runs as the target's own connection user, and + // what it creates it owns. Grants do, except around an instance data table — Windmill + // plants `custom_instance_user` grants in one, which nothing else can replay. Elsewhere + // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access. + let no_acl = is_instance_datatable_source(&db, &w_id, &req.target).await? + || is_instance_datatable_source(&db, &w_id, &req.source).await?; + + let dump_file = pg_dump_database( + &source_pg, + PgDumpOptions { schema_only, no_owner: true, no_acl, ..Default::default() }, + ) + .await?; pg_import_dump(&target_pg, &dump_file).await?; Ok(format!( @@ -3089,7 +3725,11 @@ async fn export_pg_schema( Json(req): Json, ) -> Result { let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let dump_file = pg_dump_database(&pg, true, &[]).await?; + let dump_file = pg_dump_database( + &pg, + PgDumpOptions { schema_only: true, ..Default::default() }, + ) + .await?; tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) @@ -3143,6 +3783,15 @@ async fn edit_ducklake_config( } let mut tx = db.begin().await?; + windmill_common::lock_instance_databases( + &mut tx, + new_config.settings.ducklakes.values().filter_map(|lake| { + (lake.catalog.resource_type + == windmill_common::workspaces::DucklakeCatalogResourceType::Instance) + .then_some(lake.catalog.resource_path.as_str()) + }), + ) + .await?; let args_for_audit = format!("{:?}", new_config.settings); audit_log( @@ -3218,24 +3867,54 @@ async fn edit_ducklake_config( Ok(format!("Edit ducklake config for workspace {}", &w_id)) } +/// What a save left behind. `stranded_references` names the data tables in other workspaces that +/// were governed by one this save deleted — a field rather than a sentence in a success string, +/// so the UI decides whether to warn on the data rather than on the server's prose. +#[derive(Serialize)] +pub struct EditDataTableConfigResult { + #[serde(skip_serializing_if = "Vec::is_empty")] + stranded_references: Vec, +} + +#[derive(Serialize)] +pub struct StrandedReference { + workspace_id: String, + datatable: String, +} + async fn edit_datatable_config( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, .. }: ApiAuthed, Json(mut new_config): Json, -) -> Result { +) -> JsonResult { require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &authed).await.is_ok(); let mut tx = db.begin().await?; + windmill_common::lock_instance_databases( + &mut tx, + new_config.settings.datatables.values().filter_map(|dt| { + dt.database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance) + .map(|d| d.resource_path.as_str()) + }), + ) + .await?; + // Read under the row lock this transaction will write with. `permissions`, `reference` and + // `forked_from` are carried across from what this read returns, so a permissions save + // committing between the read and the whole-document write below would be silently rolled back + // by it. let old_datatables: HashMap = serde_json::from_value( sqlx::query_scalar!( - "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + "SELECT ws.datatable->'datatables' FROM workspace_settings ws + WHERE ws.workspace_id = $1 FOR UPDATE", &w_id ) - .fetch_one(&db) + .fetch_one(&mut *tx) .await? .unwrap_or(serde_json::Value::Null), ) @@ -3257,6 +3936,58 @@ async fn edit_datatable_config( crate::datatable_migrations::validate_datatable_path_segment(&r.from)?; crate::datatable_migrations::validate_new_datatable_name(&r.to)?; } + // A rename is a claim about what this save is doing, and other workspaces' pointers are + // rewritten from it — so the claim has to match the configuration it describes, or a caller + // can move every fork of one data table onto another by asserting a rename that did not + // happen. The shape below is what "these old keys became those new keys" actually means. + { + let old_keys = &old_datatables; + let new_keys = &new_config.settings.datatables; + let froms: std::collections::HashSet<&str> = + new_config.renames.iter().map(|r| r.from.as_str()).collect(); + let tos: std::collections::HashSet<&str> = + new_config.renames.iter().map(|r| r.to.as_str()).collect(); + if froms.len() != new_config.renames.len() { + return Err(Error::BadRequest( + "A data table is renamed twice in one save".to_string(), + )); + } + if tos.len() != new_config.renames.len() { + return Err(Error::BadRequest( + "Two data tables are renamed to the same name in one save".to_string(), + )); + } + for r in &new_config.renames { + if !old_keys.contains_key(&r.from) { + return Err(Error::BadRequest(format!( + "Cannot rename data table '{}': this workspace has no such data table", + r.from + ))); + } + if !new_keys.contains_key(&r.to) { + return Err(Error::BadRequest(format!( + "Cannot rename data table '{}' to '{}': the save does not contain '{}'", + r.from, r.to, r.to + ))); + } + // The source has to be gone, or gone-and-reoccupied by another rename — which is what + // a swap is. Without this, `main -> decoy` passes against a save that keeps both, and + // every fork of `main` silently follows onto a different data table. + if new_keys.contains_key(&r.from) && !tos.contains(r.from.as_str()) { + return Err(Error::BadRequest(format!( + "Data table '{}' is renamed to '{}' but the save still contains '{}'", + r.from, r.to, r.from + ))); + } + // And the target has to be free, or freed by another rename. + if old_keys.contains_key(&r.to) && !froms.contains(r.to.as_str()) { + return Err(Error::BadRequest(format!( + "Cannot rename data table '{}' to '{}': '{}' already exists", + r.from, r.to, r.to + ))); + } + } + } // Map new name -> old name so a renamed data table inherits the previous // flag instead of being treated as brand new. @@ -3269,15 +4000,76 @@ async fn edit_datatable_config( // Migrations opt-in is owned by the enable/disable endpoints, not this config // form: preserve each existing data table's flag, and default brand-new data // tables to enabled. + // Counted here rather than after the write because this is where a rename is + // still distinguishable from a creation; emitted once the commit lands. + let mut created_substrates: Vec<&'static str> = Vec::new(); for (name, dt) in new_config.settings.datatables.iter_mut() { let lookup = rename_src .get(name.as_str()) .copied() .unwrap_or(name.as_str()); - dt.migrations_enabled = match old_datatables.get(lookup) { + let old = old_datatables.get(lookup); + dt.migrations_enabled = match old { Some(old) => old.migrations_enabled, - None => Some(true), + None => { + // Keyed by how the substrate is serialized into `workspace_settings`, + // so these line up with the `datatable_configured` adoption counts. + created_substrates.push(match dt.database.as_ref().map(|d| d.resource_type) { + Some(DataTableCatalogResourceType::Instance) => "instance", + Some(DataTableCatalogResourceType::Postgresql) => "postgresql", + None => "reference", + }); + Some(true) + } }; + // Carried across from the stored entry rather than taken from the request. `permissions` + // is an access decision, edited through its own endpoint; `reference` is what makes a fork + // answer to the workspace that governs its data table, and letting a save clear it would + // hand the fork the database outright. `forked_from` is the clone stamp the fork flow + // writes: whether an entry has one is carried the same way, since it is what marks the + // database droppable, but the schema baseline inside it is the diff view's to advance. + // `governed_by` is a clone's `reference` for its roles, and clearing it would hand the + // fork the copied rows the same way. + dt.permissions = old.and_then(|old| old.permissions.clone()); + dt.reference = old.and_then(|old| old.reference.clone()); + dt.governed_by = old.and_then(|old| old.governed_by.clone()); + dt.forked_from = match old.and_then(|old| old.forked_from.as_ref()) { + Some(stored) => Some(dt.forked_from.take().unwrap_or_else(|| stored.clone())), + None => None, + }; + // A clone's roles and grants were replayed into the database it was copied into, and hold + // for that database alone: whoever saves, it stays where it is. + if dt.governed_by.is_some() + && !crate::datatable_clone::same_database( + &dt.database, + &old.and_then(|old| old.database.clone()), + ) + { + return Err(Error::BadRequest(format!( + "Data table '{name}' is a clone taking its roles from the data table it was copied \ + from, and its grants hold for its own database only: it cannot be moved." + ))); + } + // Carrying the block onto a resource-backed entry would produce a data table the chokepoint + // refuses on every job — a save that succeeds and breaks everything afterwards. Refuse it + // instead: turning roles off first is one step, and it keeps discarding an access decision + // something somebody chose rather than a side effect of moving a database. + if dt.permissions.is_some() + && dt + .database + .as_ref() + .is_some_and(|d| d.resource_type != DataTableCatalogResourceType::Instance) + { + return Err(Error::BadRequest(format!( + "Data table '{name}' is under roles, which only a data table on the instance \ + database can be. Turn its roles off before moving it to a PostgreSQL resource." + ))); + } + // A pointer names no database of its own, so the form's empty `database` is correct there. + if dt.reference.is_some() { + dt.database = None; + } + windmill_common::workspaces::validate_datatable_shape(name, dt)?; } let args_for_audit = format!("{:?}", new_config.settings); @@ -3292,16 +4084,32 @@ async fn edit_datatable_config( ) .await?; - // Check that non-superadmins are not abusing Instance databases + // Check that non-superadmins are not abusing Instance databases, which reach a database this + // workspace does not own. Pointing an entry at another workspace's data table is not checked + // here because it cannot be requested at all: `reference` is overwritten from the stored entry + // above, for every caller. + // + // Compared against the entry the carried fields came from, not the one stored under the same + // name: otherwise swapping two names keeps each database in place while moving a clone's + // `governed_by` off the copy it governs. if !is_superadmin { for (name, dt) in new_config.settings.datatables.iter() { - if dt.database.resource_type == DataTableCatalogResourceType::Instance { - let old_dt = old_datatables.get(name); - if old_dt.is_none() - || old_dt.unwrap().database.resource_type - != DataTableCatalogResourceType::Instance - || old_dt.unwrap().database.resource_path != dt.database.resource_path - { + let old_dt = old_datatables.get( + rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()), + ); + if dt + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) + { + let unchanged = old_dt.and_then(|o| o.database.as_ref()).is_some_and(|o| { + o.resource_type == DataTableCatalogResourceType::Instance + && Some(&o.resource_path) == dt.database.as_ref().map(|d| &d.resource_path) + }); + if !unchanged { return Err(Error::BadRequest( "Only superadmins can create or modify data tables with Instance databases" .to_string(), @@ -3311,6 +4119,85 @@ async fn edit_datatable_config( } } + // Worked out from the locked entries rather than taken from `deleted_datatables`: a settings + // sync sends the whole map without that list, and dropping a governing entry strands every + // fork pointing at it all the same. + let removed: Vec = old_datatables + .keys() + .filter(|name| { + !new_config.settings.datatables.contains_key(*name) + && !new_config.renames.iter().any(|r| &r.from == *name) + }) + .cloned() + .collect(); + + // A database under roles is reached only through an entry that carries them. Roles follow an + // entry through a declared rename alone, and a settings sync never declares one, so an entry + // without roles that newly points at such a database — a name added, or an existing one + // repointed — would answer everyone there as `admin`. That holds whichever workspace governs it. + let newly_pointed: Vec<(&String, &str)> = new_config + .settings + .datatables + .iter() + // A clone carries its roles through `governed_by` rather than `permissions`. + .filter(|(_, dt)| dt.permissions.is_none() && dt.governed_by.is_none()) + .filter_map(|(name, dt)| { + let db = dt + .database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?; + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + let repointed = old_datatables + .get(lookup) + .and_then(|old| old.database.as_ref()) + .is_none_or(|old_db| { + old_db.resource_type != db.resource_type + || old_db.resource_path != db.resource_path + }); + repointed.then_some((name, db.resource_path.as_str())) + }) + .collect(); + // Another workspace turning roles on for the same database holds only its own settings row, so + // without this the scan below could read past its uncommitted write. + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + newly_pointed.iter().map(|(_, dbname)| *dbname), + ) + .await?; + let governed_elsewhere: Vec = if newly_pointed.is_empty() { + vec![] + } else { + sqlx::query_scalar( + "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id <> $1 AND (dt.value ? 'permissions' OR dt.value ? 'governed_by') + AND dt.value->'database'->>'resource_type' = 'instance'", + ) + .bind(&w_id) + .fetch_all(&mut *tx) + .await? + }; + for (name, dbname) in newly_pointed { + let governed_here = old_datatables.values().any(|old| { + (old.permissions.is_some() || old.governed_by.is_some()) + && old.database.as_ref().is_some_and(|d| { + d.resource_type == DataTableCatalogResourceType::Instance + && d.resource_path == dbname + }) + }); + if governed_here || governed_elsewhere.iter().any(|g| g == dbname) { + return Err(Error::BadRequest(format!( + "Data table '{name}' would point at database '{dbname}', which a data table under \ + roles uses, without carrying those roles: everyone reaching '{name}' would connect \ + there as `admin`. Rename the data table under roles from the data table settings, \ + which carries its roles, or turn its roles off first." + ))); + } + } + let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -3332,8 +4219,53 @@ async fn edit_datatable_config( ) .await?; + // A fork points at a data table by name, so a rename here has to follow or every fork's entry + // resolves to nothing. In two passes through a temporary name, like the migration cascade one + // layer down: applied in order, `sa -> sb` then `sb -> sa` would move what pointed at `sa` all + // the way back to `sa`, and `A -> B`, `B -> C` would carry `A`'s pointers to `C`. Each pointer + // moves once, from what it named before this save. Inside the transaction: the rename and the + // pointers that name it are one change, and half of it is a fork whose jobs stop. + for (i, r) in new_config.renames.iter().enumerate() { + repoint_datatable_references(&mut tx, &w_id, &r.from, &format!("__wm_rename_tmp/{i}")) + .await?; + } + for (i, r) in new_config.renames.iter().enumerate() { + repoint_datatable_references(&mut tx, &w_id, &format!("__wm_rename_tmp/{i}"), &r.to) + .await?; + } + + // A deletion cannot be followed the same way — there is nothing to point at any more. Read who + // is left stranded so the caller is told, the way deleting a workspace does. + let mut stranded: Vec = Vec::new(); + for name in &removed { + let rows = sqlx::query!( + r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE EXISTS ( + SELECT 1 FROM (VALUES ('reference'), ('governed_by')) k(link) + WHERE dt.value->k.link->>'workspace_id' = $1 + AND dt.value->k.link->>'datatable' = $2 + )"#, + &w_id, + name, + ) + .fetch_all(&mut *tx) + .await?; + stranded.extend( + rows.into_iter().map(|r| StrandedReference { + workspace_id: r.workspace_id, + datatable: r.datatable, + }), + ); + } + tx.commit().await?; + for substrate in created_substrates { + windmill_common::feature_usage::log_feature_usage("datatable", "created", substrate); + } + crate::datatable_migrations::record_datatable_cascade_deployments( &authed, &db, @@ -3342,7 +4274,9 @@ async fn edit_datatable_config( ) .await?; - Ok(format!("Edit datatable config for workspace {}", &w_id)) + Ok(Json(EditDataTableConfigResult { + stranded_references: stranded, + })) } #[derive(Deserialize)] @@ -3437,54 +4371,6 @@ fn cleanup_legacy_git_sync_settings_in_memory( #[cfg(not(feature = "enterprise"))] const CE_GIT_SYNC_MAX_USERS: i64 = 2; -/// Auto-pull is licensed per plan, not just per build: the poller only serves -/// Enterprise plans at runtime, so the save path must reject the setting too — -/// otherwise an EE binary without the plan could still register a webhook and -/// receive webhook-driven pulls. -#[cfg(feature = "enterprise")] -async fn check_git_sync_ee_license(feature: &str) -> Result<()> { - if !matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ) { - return Err(Error::BadRequest(format!( - "{feature} requires an Enterprise license" - ))); - } - Ok(()) -} - -#[cfg(feature = "enterprise")] -async fn check_auto_pull_license() -> Result<()> { - check_git_sync_ee_license("Automatic pull from git").await -} - -/// In-app PR creation (promotion/fork deploy branches) drives GitHub API calls -/// from the deploy completion hook; runtime-gate it like auto-pull. -#[cfg(feature = "enterprise")] -async fn check_open_prs_license<'a>( - mut repos: impl Iterator, -) -> Result<()> { - if repos.any(|r| r.promotion_open_prs || r.fork_open_prs) { - check_git_sync_ee_license("Opening pull requests from Windmill").await?; - } - Ok(()) -} - -/// Promotion mode (`use_individual_branch`: per-item `wm_deploy/**` deploy -/// branches) is an EE feature; runtime-gate it like auto-pull and PR creation -/// so an enterprise binary without an active plan can't enable it via either -/// git-sync edit endpoint. -#[cfg(feature = "enterprise")] -async fn check_promotion_license<'a>( - mut repos: impl Iterator, -) -> Result<()> { - if repos.any(|r| r.use_individual_branch.unwrap_or(false)) { - check_git_sync_ee_license("Promotion mode").await?; - } - Ok(()) -} - /// Promotion on a dev workspace needs the dev-aware sync script (hub >= 28796): /// an older pinned script bundles a CLI that force-disables per-item branches /// on every fork, so enabling promotion would silently keep deploying to the @@ -3713,9 +4599,10 @@ async fn edit_git_sync_config( // stay clean. for repo in git_sync_settings.repositories.iter_mut() { if let Some(ap) = repo.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } repo.open_pr_error = None; + repo.credential = None; } reject_parent_only_git_sync_settings_on_fork( &db, @@ -3735,18 +4622,6 @@ async fn edit_git_sync_config( )); } #[cfg(feature = "enterprise")] - if git_sync_settings - .repositories - .iter() - .any(|r| r.auto_pull.as_ref().is_some_and(|a| a.enabled)) - { - check_auto_pull_license().await?; - } - #[cfg(feature = "enterprise")] - check_open_prs_license(git_sync_settings.repositories.iter()).await?; - #[cfg(feature = "enterprise")] - check_promotion_license(git_sync_settings.repositories.iter()).await?; - #[cfg(feature = "enterprise")] check_dev_promotion_script_version(&db, &w_id, git_sync_settings.repositories.iter()) .await?; #[cfg(all(feature = "enterprise", feature = "private"))] @@ -3816,6 +4691,7 @@ async fn edit_git_sync_config( continue; }; repo.open_pr_error = old.open_pr_error.clone(); + repo.credential = old.credential.clone(); if let (Some(new_ap), Some(old_ap)) = (repo.auto_pull.as_mut(), old.auto_pull.as_ref()) { @@ -3861,6 +4737,7 @@ async fn edit_git_sync_config( .flatten() .and_then(|v| serde_json::from_value(v).ok()); let removed_webhooks: Vec<(String, i64)> = existing + .as_ref() .map(|e| { e.repositories .iter() @@ -3894,6 +4771,19 @@ async fn edit_git_sync_config( // `sync_repo_webhook` writes back the webhook fields it changes itself: // the remote hook and the record of it have to move together, so // persisting them out here would let one land without the other. + // Before the webhook reconcile, which decides whether this repo can have + // one from the credential this records. Also puts a short-lived or + // under-scoped token in front of the operator while they are still on the + // settings page, rather than when it expires. + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); @@ -3944,9 +4834,10 @@ async fn edit_git_sync_repository( // existing repo re-derives it from the DB (carried over below) and a new one // starts clean. if let Some(ap) = new_config.repository.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } new_config.repository.open_pr_error = None; + new_config.repository.credential = None; reject_parent_only_git_sync_settings_on_fork( &db, &w_id, @@ -3968,19 +4859,6 @@ async fn edit_git_sync_repository( )); } #[cfg(feature = "enterprise")] - if new_config - .repository - .auto_pull - .as_ref() - .is_some_and(|a| a.enabled) - { - check_auto_pull_license().await?; - } - #[cfg(feature = "enterprise")] - check_open_prs_license(std::iter::once(&new_config.repository)).await?; - #[cfg(feature = "enterprise")] - check_promotion_license(std::iter::once(&new_config.repository)).await?; - #[cfg(feature = "enterprise")] check_dev_promotion_script_version(&db, &w_id, std::iter::once(&new_config.repository)).await?; #[cfg(all(feature = "enterprise", feature = "private"))] check_dev_promotion_targets_parent_repo(&db, &w_id, std::iter::once(&new_config.repository)) @@ -4071,6 +4949,7 @@ async fn edit_git_sync_repository( // from the UI cannot revert what the poller/webhook layer wrote. let mut updated = new_config.repository; updated.open_pr_error = existing_repo.open_pr_error.clone(); + updated.credential = existing_repo.credential.clone(); match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) { (Some(new_ap), Some(old_ap)) => { new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); @@ -4085,13 +4964,6 @@ async fn edit_git_sync_repository( } _ => {} } - // The request-side license gate above only saw the submitted config; the - // preservation can resurrect an enabled auto_pull (None arm), so re-check - // the effective state before it gets written and reconciled. - #[cfg(feature = "enterprise")] - if updated.auto_pull.as_ref().is_some_and(|a| a.enabled) { - check_auto_pull_license().await?; - } *existing_repo = updated; } else { // Repository doesn't exist, add it as a new repository @@ -4128,6 +5000,15 @@ async fn edit_git_sync_repository( .iter_mut() .find(|r| r.git_repo_resource_path == new_config.git_repo_resource_path) { + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); } @@ -4267,6 +5148,10 @@ async fn delete_git_sync_repository( } } + // The stored credential is deliberately left alone: it belongs to the git + // repository resource, which this endpoint does not delete, and the resource + // still authenticates with it for connection tests and commit lookups. + // Trigger git sync for repository deletion handle_deployment_metadata( &authed.email, @@ -4373,6 +5258,159 @@ async fn edit_default_app( )); } +#[derive(Deserialize)] +struct EditGuestAccess { + guest_access_enabled: bool, +} + +/// Turn guest sessions on or off for this workspace. Off by default, and off is +/// authoritative and immediate: the switch is re-read where a guest session is +/// minted (`guest_app_admits`) and at the auth door on every guest request, so an app +/// whose policy already says `guest` — pushed by git-sync, say — closes to guests on +/// the next request, sessions already issued included. +async fn edit_guest_access( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(EditGuestAccess { guest_access_enabled }): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if guest_access_enabled { + windmill_common::workspaces::require_guest_support()?; + } + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace_settings SET guest_access_enabled = $1 WHERE workspace_id = $2", + guest_access_enabled, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_guest_access", + ActionKind::Update, + &w_id, + Some(&guest_access_enabled.to_string()), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!( + "Guest access set to {guest_access_enabled} for workspace {w_id}" + )) +} + +#[derive(Deserialize)] +struct EditAddAdminsAndDevelopersToForks { + add_admins_and_developers_to_forks: bool, +} + +async fn edit_add_admins_and_developers_to_forks( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(EditAddAdminsAndDevelopersToForks { add_admins_and_developers_to_forks }): Json< + EditAddAdminsAndDevelopersToForks, + >, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace_settings SET add_admins_and_developers_to_forks = $1 WHERE workspace_id = $2", + add_admins_and_developers_to_forks, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_add_admins_and_developers_to_forks", + ActionKind::Update, + &w_id, + Some(&add_admins_and_developers_to_forks.to_string()), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!( + "Adding admins and developers to new forks set to {add_admins_and_developers_to_forks} for workspace {w_id}" + )) +} + +#[derive(Deserialize)] +struct EditGuestJwtKey { + /// A PEM public key (RS or ES family), or a JWKS URL, at most one. Both empty clears the + /// workspace key; verification then falls back to the instance issuer (`JWT_EXT_JWKS_URL`) + /// off cloud, or refuses the JWT if none is set. The off-switch is `guest_access_enabled`. + public_key: Option, + jwks_url: Option, +} + +/// Configure the key a guest JWT (`jwt_guest_`) is verified against for this workspace. +/// Workspace-admin gated, like the guest switch: guests are free up to the instance +/// allowance on any plan, so configuring their key needs no licence. The key is +/// validated before it is stored so a typo is refused here, not silently on every guest +/// later: a PEM must parse as an RS/ES public key (HS* has no PEM form and is +/// unreachable), and a JWKS URL must be fetchable and hold at least one usable signing key. +async fn edit_guest_jwt_key( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(EditGuestJwtKey { public_key, jwks_url }): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let public_key = public_key.filter(|s| !s.trim().is_empty()); + let jwks_url = jwks_url.filter(|s| !s.trim().is_empty()); + if public_key.is_some() && jwks_url.is_some() { + return Err(Error::BadRequest( + "Set a PEM public key or a JWKS URL, not both".to_string(), + )); + } + // Clearing stays allowed wherever guests are: a key nobody can use is still worth + // removing. + if public_key.is_some() || jwks_url.is_some() { + windmill_common::workspaces::require_guest_support()?; + } + if let Some(pem) = public_key.as_deref() { + windmill_common::guest_jwt::decoding_key_from_pem(pem)?; + } + if let Some(url) = jwks_url.as_deref() { + windmill_common::guest_jwt::fetch_jwks(url).await?; + } + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace_settings SET guest_jwt_public_key = $1, guest_jwt_jwks_url = $2 WHERE workspace_id = $3", + public_key, + jwks_url, + &w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "workspaces.edit_guest_jwt_key", + ActionKind::Update, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Guest JWT key updated for workspace {w_id}")) +} + async fn edit_default_scripts( authed: ApiAuthed, Extension(db): Extension, @@ -5012,11 +6050,19 @@ async fn set_encryption_key( )); } - // Build the previous cipher before the transaction (reads from cache/pool) - let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; - let mut tx = db.begin().await?; + // Under the row's lock, so two rotations racing serialize and each sees the key the + // other committed. The AI session backups in the workspace storage live under a prefix + // named by a generation this bumps (with the key, in this transaction) rather than + // being re-keyed; the older generations are deleted once this one has committed (see + // `ai_session_backups`). The same key set again is no rotation to them. + let previous_key: String = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud' FOR UPDATE", + ) + .bind(&w_id) + .fetch_one(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), @@ -5024,18 +6070,27 @@ async fn set_encryption_key( ) .execute(&mut *tx) .await?; + let backups_generation: Option = if previous_key != request.new_key { + sqlx::query_scalar( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 WHERE workspace_id = $1 \ + RETURNING ai_sessions_backup_generation", + ) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + } else { + None + }; + + // From the keys read and written under the lock, never from `build_crypt`: its + // cache can still hold a key an earlier rotation replaced, and the git-sync + // secrets below are skipped rather than failed when they do not decrypt. + let previous_encryption_key = crypt_from_key_with_suffix(&previous_key, ""); + let new_encryption_key = crypt_from_key_with_suffix(&request.new_key, ""); let mut reencrypted_secret_paths: Vec = Vec::new(); if !request.skip_reencrypt.unwrap_or(false) { - // Build the new cipher directly from the key string, since the transaction - // hasn't committed yet and build_crypt() would read the old key from the pool. - let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", request.new_key, salt) - } else { - request.new_key.clone() - }; - let new_encryption_key = magic_crypt::new_magic_crypt!(crypt_key, 256); - let mut truncated_new_key = request.new_key.clone(); truncated_new_key.truncate(8); tracing::warn!( @@ -5075,11 +6130,28 @@ async fn set_encryption_key( } } + reencrypt_git_sync_secrets( + &mut tx, + &w_id, + &previous_encryption_key, + &new_encryption_key, + ) + .await?; + tx.commit().await?; // Invalidate the cache only after the transaction has committed WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Nothing writes under the older generations any more; the browsers push their + // sessions again under the new one. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_older(db.clone(), w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Build the batch: one event for the encryption key itself plus one per // re-encrypted secret variable. The batch entrypoint dispatches a single // git-sync job per repo carrying all items, so repos with Secrets sync @@ -5103,6 +6175,64 @@ async fn set_encryption_key( return Ok(()); } +/// Move the git-sync secrets the server keeps under the workspace key (stored +/// repository tokens, webhook secrets) to the new key. They are never synced, so +/// unlike variables they are still under the old key when the caller skips +/// re-encryption. +async fn reencrypt_git_sync_secrets( + conn: &mut sqlx::PgConnection, + w_id: &str, + old: &magic_crypt::MagicCrypt256, + new: &magic_crypt::MagicCrypt256, +) -> Result<()> { + let Some((mut credentials, mut git_sync)) = + sqlx::query_as::<_, (serde_json::Value, Option)>( + "SELECT git_credentials, git_sync FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(w_id) + .fetch_optional(&mut *conn) + .await? + else { + return Ok(()); + }; + let reencrypt = |value: &mut serde_json::Value| { + let Some(ciphertext) = value.as_str() else { + return; + }; + match decrypt(old, ciphertext.to_string()) { + Ok(plain) => *value = serde_json::Value::String(encrypt(new, &plain)), + // Left by an earlier rotation and unrecoverable either way; failing here + // would block every later rotation of the workspace. + Err(e) => tracing::warn!( + "a git-sync secret of workspace {w_id} does not decrypt under its current key, leaving it as is: {e}" + ), + } + }; + for entry in credentials.as_array_mut().into_iter().flatten() { + if let Some(token) = entry.get_mut("token") { + reencrypt(token); + } + } + let repositories = git_sync + .as_mut() + .and_then(|g| g.get_mut("repositories")) + .and_then(|r| r.as_array_mut()); + for repo in repositories.into_iter().flatten() { + if let Some(secret) = repo.pointer_mut("/auto_pull/webhook_secret") { + reencrypt(secret); + } + } + sqlx::query( + "UPDATE workspace_settings SET git_credentials = $2, git_sync = $3 WHERE workspace_id = $1", + ) + .bind(w_id) + .bind(credentials) + .bind(git_sync) + .execute(&mut *conn) + .await?; + Ok(()) +} + #[derive(Serialize)] struct UsedTriggers { pub websocket_used: bool, @@ -5231,7 +6361,7 @@ async fn user_workspaces( workspace.is_dev_workspace, workspace.dev_workspace_label, workspace.owner AS \"created_by?\", CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, - usr.disabled + usr.disabled, usr.is_service_account FROM workspace JOIN usr ON usr.workspace_id = workspace.id JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id @@ -5249,6 +6379,14 @@ struct SessionWorkspaceStatusRequest { workspace_ids: Vec, } +/// `ai_config.sessions_retention_days` as stored, `None` when unset or not a count of days. +pub fn sessions_retention_days(value: Option<&serde_json::Value>) -> Option { + value + .and_then(|v| v.as_u64()) + .filter(|days| *days >= 1) + .and_then(|days| u32::try_from(days).ok()) +} + /// Reconciliation support for client-side AI sessions, which the backend cannot touch /// directly. The client posts the workspace ids its sessions reference and uses the /// per-id status to keep sessions in sync with workspace lifecycle: `deleted` (no row, or @@ -5298,6 +6436,42 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The AI session retention a browser deletes its local copies by (docs/ai-session-backups.md). +/// Its own route, not a field on the status above, whose shape an older tab still reads. Unlike +/// a status, it answers only for a workspace this caller can be authed into: a setting is the +/// workspace's to tell, so a disabled membership gets none though its sessions still reconcile. +async fn session_workspace_retention( + Extension(db): Extension, + authed: ApiAuthed, + Json(req): Json, +) -> JsonResult> { + if req.workspace_ids.len() > 1000 { + return Err(Error::BadRequest( + "Too many workspace ids (max 1000)".to_string(), + )); + } + let email = &authed.email; + let is_superadmin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; + let rows = sqlx::query!( + "SELECT workspace_settings.workspace_id AS \"id!\", + workspace_settings.ai_config->'sessions_retention_days' AS retention + FROM workspace_settings + LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2 + WHERE workspace_settings.workspace_id = ANY($1) + AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + &req.workspace_ids[..], + email, + is_superadmin, + ) + .fetch_all(&db) + .await?; + let days = rows + .into_iter() + .filter_map(|r| sessions_retention_days(r.retention.as_ref()).map(|days| (r.id, days))) + .collect(); + Ok(Json(days)) +} + /// The instance critical alert channels belong to the instance operator, who on cloud is /// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run /// throwaway copies of their parent's runnables, so instance-wide operational alerting must @@ -5694,9 +6868,8 @@ async fn clone_workspace_data( // Clone the forker's own per-user drafts (plus the legacy NULL-email // workspace draft, if any) so they keep their pending edits in the // fork. Other users' drafts are intentionally NOT cloned — they don't - // own a `usr` row in the fork (see `clone_workspace_full`) so their - // drafts would dangle and the home-page `draft_users` aggregate would - // surface them as duplicate legacy entries. + // own a `usr` row in the fork (see `clone_workspace_full`), so those + // drafts would belong to someone the fork holds no membership for. clone_drafts(tx, source_workspace_id, target_workspace_id, &authed.email).await?; // Clone workspace runnable dependencies and dependency map @@ -5760,7 +6933,7 @@ async fn clone_triggers_and_schedules( path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, edited_at, extra_perms, authentication_method, http_method, static_asset_config, is_static_website, workspaced_route, wrap_body, - raw_string, authentication_resource_path, summary, description, + raw_string, allowed_origins, authentication_resource_path, summary, description, error_handler_path, error_handler_args, retry, request_type, mode, permissioned_as, labels ) @@ -5768,7 +6941,7 @@ async fn clone_triggers_and_schedules( path, route_path, route_path_key, script_path, is_flow, $1, edited_by, edited_at, extra_perms, authentication_method, http_method, static_asset_config, is_static_website, workspaced_route, wrap_body, - raw_string, authentication_resource_path, summary, description, + raw_string, allowed_origins, authentication_resource_path, summary, description, error_handler_path, error_handler_args, retry, request_type, 'disabled'::TRIGGER_MODE, permissioned_as, labels FROM http_trigger @@ -6009,7 +7182,8 @@ async fn update_workspace_settings( ducklake = source_ws.ducklake, dbt_warehouses = source_ws.dbt_warehouses, datatable = source_ws.datatable, - git_app_installations = source_ws.git_app_installations + git_app_installations = source_ws.git_app_installations, + add_admins_and_developers_to_forks = source_ws.add_admins_and_developers_to_forks FROM workspace_settings source_ws WHERE source_ws.workspace_id = $1 AND workspace_settings.workspace_id = $2 @@ -6049,9 +7223,10 @@ async fn update_workspace_settings( // Auto-pull and fork PRs are parent-owned and must not be inherited: // the fork would otherwise carry the parent's webhook id (turning off // auto-pull on the fork would delete the parent's webhook). A fork - // still inherits the push-direction config and the installation. - // Repo → fork sync is driven by the parent's webhook/poller - // (`sync_forks`), which routes the fork's `wm-fork/**` branch into it. + // still inherits the push-direction config, the installation, and the + // recorded credential status, which describes the repository rather + // than belonging to either workspace and would otherwise leave the + // fork unqualified for managed features until its first check. r.auto_pull = None; r.fork_open_prs = false; r.open_pr_error = None; @@ -6149,14 +7324,21 @@ async fn copy_workspace_members( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, + admins_and_developers_only: bool, ) -> Result<()> { + // Admins and developers join as manual members: the fork does not inherit the source's + // instance-group config, so a copied `instance_group` provenance would let the fork's + // reconciliation delete them and their data. sqlx::query!( "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via) - SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via + SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, + CASE WHEN $3 THEN NULL ELSE added_via END FROM usr WHERE workspace_id = $2 + AND (NOT $3 OR (NOT operator AND NOT disabled AND NOT is_service_account)) ON CONFLICT DO NOTHING", target_workspace_id, source_workspace_id, + admins_and_developers_only, ) .execute(&mut **tx) .await?; @@ -6169,8 +7351,8 @@ async fn clone_resource_types( target_workspace_id: &str, ) -> Result<()> { sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset) - SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset + "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name) + SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", source_workspace_id, @@ -6372,7 +7554,7 @@ async fn clone_scripts( } /// The parsed dbt graph a deployed script carries: its models, their SQL and -/// tests, and the `ref()` lineage between them. +/// tests, and the `ref()` and column-level lineage between them. /// /// Keyed on (workspace_id, script_path, script_hash), and the fork keeps every /// script's hash, so each row moves across as itself. @@ -6390,11 +7572,11 @@ async fn clone_dbt_graph( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at) + columns, column_schema, freshness, raw_code, original_file_path, ingested_at) SELECT $2, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at + columns, column_schema, freshness, raw_code, original_file_path, ingested_at FROM dbt_node WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", source_workspace_id, @@ -6414,6 +7596,24 @@ async fn clone_dbt_graph( ) .execute(&mut **tx) .await?; + // Column lineage travels with the rest of the graph, and it has to: the + // snapshot's digest covers it, so a fork missing these rows recomputes the + // digest the source stored, matches, and stores nothing — leaving the + // lineage gone until someone redeploys, which is the failure this whole + // function exists to prevent. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind, + ingested_at) + SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column, + child_unique_id, child_column, lineage_kind, ingested_at + FROM dbt_column_edge + WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + source_workspace_id, + target_workspace_id + ) + .execute(&mut **tx) + .await?; sqlx::query!( "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest, relation_root_at_last_ingest, ingested_at) @@ -6557,6 +7757,127 @@ async fn clear_orphaned_compat_address( Ok(()) } +/// SQL boolean: the principal the `principal` expression yields resolves in the workspace bound as +/// `$1`. The same predicate `clone_scripts` and `clone_flows` inline, whose `query!` macros cannot +/// take a composed string, so keep the three in step. +fn principal_resolves_sql(principal: &str) -> String { + format!( + "CASE WHEN {principal} LIKE 'u/%' THEN EXISTS ( + SELECT 1 FROM usr u WHERE u.workspace_id = $1 + AND u.username = substring({principal} from 3) + UNION ALL + SELECT 1 FROM password pw WHERE pw.super_admin + AND (pw.username = substring({principal} from 3) + OR pw.email = substring({principal} from 3))) + WHEN {principal} LIKE 'g/%' THEN EXISTS ( + SELECT 1 FROM group_ g WHERE g.workspace_id = $1 + AND g.name = substring({principal} from 3)) + ELSE EXISTS ( + SELECT 1 FROM usr u WHERE u.workspace_id = $1 AND u.username = {principal} + UNION ALL + SELECT 1 FROM password pw WHERE pw.email = {principal} AND pw.super_admin) + END" + ) +} + +/// Re-point the identities a fork clones verbatim at its creator when they name nobody in the fork, +/// once its membership is final so copied members keep theirs. Unlike scripts and flows these +/// cannot drop the identity: an app deploy rejects a preserved one that does not resolve, and +/// publisher apps, schedules and triggers need one to run. +async fn repoint_unresolvable_cloned_identities( + tx: &mut Transaction<'_, Postgres>, + target_workspace_id: &str, + authed: &ApiAuthed, +) -> Result<()> { + let principal = username_to_permissioned_as(&authed.username); + + sqlx::query(&format!( + "UPDATE app SET policy = policy + || jsonb_build_object('on_behalf_of', $2::text, 'on_behalf_of_email', $3::text) + WHERE workspace_id = $1 AND policy->>'on_behalf_of' IS NOT NULL + AND NOT ({})", + principal_resolves_sql("(policy->>'on_behalf_of')") + )) + .bind(target_workspace_id) + .bind(&principal) + .bind(&authed.email) + .execute(&mut **tx) + .await?; + + // A draft holding a genuine NUL escape (the rule of `json_text_has_nul_escape`) is left as it + // is, since parsing it would abort the fork. The check must stay in a CASE: json `->>` raises on + // a NUL anywhere in the value, and Postgres reorders plain AND conditions. + let nul_escape = r"(^|[^\\])(\\\\)*\\u0000"; + sqlx::query(&format!( + "UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), + ARRAY['policy', 'on_behalf_of'], to_jsonb($2::text)), + ARRAY['policy', 'on_behalf_of_email'], to_jsonb($3::text))) + WHERE workspace_id = $1 AND typ IN ('app', 'raw_app') + AND CASE WHEN value::text ~ $4 THEN false + ELSE value->'policy'->>'on_behalf_of' IS NOT NULL AND NOT ({}) END", + principal_resolves_sql("(value->'policy'->>'on_behalf_of')") + )) + .bind(target_workspace_id) + .bind(&principal) + .bind(&authed.email) + .bind(nul_escape) + .execute(&mut **tx) + .await?; + + sqlx::query(&format!( + "UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), + ARRAY['permissioned_as'], to_jsonb($2::text))) + WHERE workspace_id = $1 AND starts_with(typ::text, 'trigger_') + AND CASE WHEN value::text ~ $3 THEN false + ELSE value->>'permissioned_as' IS NOT NULL AND NOT ({}) END", + principal_resolves_sql("(value->>'permissioned_as')") + )) + .bind(target_workspace_id) + .bind(&principal) + .bind(nul_escape) + .execute(&mut **tx) + .await?; + + let column_resolves = principal_resolves_sql("permissioned_as"); + + // SAFETY: every table name is a literal from this list, never user input. + for table in [ + "http_trigger", + "websocket_trigger", + "kafka_trigger", + "nats_trigger", + "postgres_trigger", + "mqtt_trigger", + "amqp_trigger", + "sqs_trigger", + "gcp_trigger", + "azure_trigger", + "email_trigger", + ] { + sqlx::query(&format!( + "UPDATE {table} SET permissioned_as = $2 + WHERE workspace_id = $1 AND NOT ({column_resolves})" + )) + .bind(target_workspace_id) + .bind(&principal) + .execute(&mut **tx) + .await?; + } + + // `email` is still written for workers that predate `permissioned_as`. + sqlx::query(&format!( + "UPDATE schedule SET permissioned_as = $2, email = $3 + WHERE workspace_id = $1 AND NOT ({column_resolves})" + )) + .bind(target_workspace_id) + .bind(&principal) + .bind(&authed.email) + .execute(&mut **tx) + .await?; + + Ok(()) +} + /// Carries over the recorded principal under the rule spelled out on [`clone_scripts`]. async fn clone_flows( tx: &mut Transaction<'_, Postgres>, @@ -7002,14 +8323,32 @@ async fn clone_drafts( // filtered like `clone_scripts`: the address the draft still carries re-derives the // clone's own principal at deploy time, which is the more accurate answer of the two. sqlx::query!( - "INSERT INTO draft (workspace_id, path, typ, value, created_at, email) + // A script hash is content-addressed and copied as-is, so a script draft's base + // still names a version the clone has. `clone_flows` / `clone_apps` mint new ids, + // so those drafts arrive with no base (staleness falls back to the timestamps), + // lineage field included, or the next autosave would re-derive the source id. + // + // `clean` is `strip_json_nul`'s parity rule in SQL, so a pre-sanitizer U+0000 + // escape cannot abort the clone on `to_jsonb` or arrive with its principal + // unstripped: escaped backslashes park on chr(1) (lossless, a `json` value's text + // cannot hold a raw control byte) so only a real NUL is removed, and chr(92) + // spells the backslash so no escape sequence reaches this source file. + r#"INSERT INTO draft (workspace_id, path, typ, value, created_at, email, base) SELECT $2, path, typ, - CASE WHEN typ IN ('script', 'flow') - THEN to_json(to_jsonb(value) - 'on_behalf_of') - ELSE value END, - created_at, email - FROM draft - WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", + to_json( + CASE WHEN typ IN ('script', 'flow') THEN clean - 'on_behalf_of' ELSE clean END + - CASE WHEN typ = 'flow' THEN 'version_id' + WHEN typ IN ('app', 'raw_app') THEN 'parent_version' + ELSE '' END + ), + created_at, email, + CASE WHEN typ = 'script' THEN base END + FROM ( + SELECT d.path, d.typ, d.created_at, d.email, d.base, + replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean + FROM draft d + WHERE d.workspace_id = $1 AND (d.email = $3 OR d.email IS NULL) + ) s"#, source_workspace_id, target_workspace_id, authed_email, @@ -7027,8 +8366,8 @@ async fn clone_workspace_runnable_dependencies( ) -> Result<()> { // Clone workspace_runnable_dependencies sqlx::query!( - "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) - SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path) + SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path FROM workspace_runnable_dependencies WHERE workspace_id = $2", target_workspace_id, @@ -7037,7 +8376,16 @@ async fn clone_workspace_runnable_dependencies( .execute(&mut **tx) .await?; - // Clone dependency_map to preserve import relationships + // Recorded so the clone's own relocks have something to match; with no row they record NULL + // and nothing in it ever skips. Hashed from the locks the clone holds rather than copied from + // the source's rows, which are only as current as the last write to them: one left stale by a + // supplied lock deployed before this was recorded names a lock the clone no longer has, and an + // importer that resolved against the real one would then skip a relock it needed. + record_lock_hashes_for_workspace(tx, target_workspace_id).await?; + + // Deliberately without `imported_lockfile_hash`: it records what an importer resolved against + // when it was last locked, which nothing here can establish for the version the clone got. + // Left NULL, every importer relocks once and re-anchors both sides to what the clone holds. sqlx::query!( "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) SELECT $1, importer_path, importer_kind, imported_path, importer_node_id @@ -7144,6 +8492,7 @@ async fn create_workspace_fork_branch( // dangling branch on the synced repos. check_fork_w_id_conflict(&db, &nw.id).await?; purge_stale_fork_diff_state(&db, &nw.id).await?; + validate_forked_datatables(&db, &authed, &w_id, &nw).await?; Ok(Json( handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, @@ -7176,12 +8525,142 @@ async fn snapshot_datatable_schema( .map_err(|e| Error::internal_err(format!("Failed to serialize schema: {}", e))) } -async fn apply_forked_datatable( - db: &DB, +/// Turn every data table the fork chose to keep into a pointer at the parent's entry. +/// +/// `clone_workspace_data` copies `workspace_settings` wholesale, so a kept data table arrives as a +/// byte-identical copy naming the parent's database — including the parent's `permissions`, which +/// a fork admin could then edit to widen their own access to it. A pointer has nothing local to +/// edit: the parent's entry stays the only place the decision lives. +/// +/// The cloned data tables are skipped: they own a fresh database of their own, and +/// `apply_forked_datatable` has already dropped their copied `permissions` — for a clone of a data +/// table under roles, in favor of `governed_by`. +async fn point_kept_datatables_at_parent( tx: &mut Transaction<'_, Postgres>, parent_w_id: &str, forked_w_id: &str, + cloned: &[ForkedDatatableInfo], +) -> Result<()> { + let settings: Option = sqlx::query_scalar!( + "SELECT datatable FROM workspace_settings WHERE workspace_id = $1", + forked_w_id + ) + .fetch_optional(&mut **tx) + .await? + .flatten(); + + let Some(mut settings) = settings else { + return Ok(()); + }; + let Some(datatables) = settings + .get_mut("datatables") + .and_then(|d| d.as_object_mut()) + else { + return Ok(()); + }; + + let mut changed = false; + for (name, entry) in datatables.iter_mut() { + if cloned.iter().any(|c| &c.name == name) { + continue; + } + let dt: DataTable = match serde_json::from_value(entry.clone()) { + Ok(dt) => dt, + Err(_) => continue, + }; + // Already a pointer: the parent was itself a fork, and its entry names the workspace that + // governs. Following it from here is the same answer, so leave it alone. + if dt.reference.is_some() { + continue; + } + // Only instance databases. A resource-backed data table names a resource, and the settings + // clone gave the fork its own copy of that resource in its own workspace — pointing at the + // parent's entry would silently move the fork onto the parent's resource instead. + if dt + .database + .as_ref() + .is_none_or(|d| d.resource_type != DataTableCatalogResourceType::Instance) + { + continue; + } + *entry = serde_json::to_value(DataTable { + database: None, + reference: Some(windmill_common::workspaces::DataTableReference { + workspace_id: parent_w_id.to_string(), + datatable: name.clone(), + }), + governed_by: None, + forked_from: None, + migrations_enabled: dt.migrations_enabled, + permissions: None, + }) + .map_err(|e| Error::internal_err(format!("serializing data table '{name}': {e}")))?; + changed = true; + } + + if changed { + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + settings, + forked_w_id + ) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +/// Move every pointer, and every clone's `governed_by`, in any workspace that names `(w_id, from)` +/// to `(w_id, to)`. +/// +/// `EXISTS` rather than a `LIKE` over the whole document: the update rewrites the row, so matching +/// every workspace that holds any pointer would rewrite rows to a byte-identical value and hold an +/// exclusive lock on them until commit. +async fn repoint_datatable_references( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + from: &str, + to: &str, +) -> Result<()> { + sqlx::query!( + r#"UPDATE workspace_settings ws + SET datatable = ( + SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg( + dt.key, + (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $1 + AND r.v->'governed_by'->>'datatable' = $2 + THEN jsonb_set(r.v, '{governed_by,datatable}', to_jsonb($3::text)) + ELSE r.v END + FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $1 + AND dt.value->'reference'->>'datatable' = $2 + THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text)) + ELSE dt.value END AS v) r) + )) + FROM jsonb_each(ws.datatable->'datatables') dt + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d, + LATERAL (VALUES ('reference'), ('governed_by')) k(link) + WHERE d.value->k.link->>'workspace_id' = $1 + AND d.value->k.link->>'datatable' = $2 + )"#, + w_id, + from, + to, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn apply_forked_datatable( + db: &DB, + tx: &mut Transaction<'_, Postgres>, + authed: &ApiAuthed, + parent_w_id: &str, + forked_w_id: &str, fdt: &ForkedDatatableInfo, + copy: &crate::datatable_clone::MadeCopy, ) -> Result<()> { windmill_common::validate_dbname(&fdt.new_dbname)?; if !fdt.new_dbname.starts_with("wm_fork_") { @@ -7190,6 +8669,52 @@ async fn apply_forked_datatable( fdt.new_dbname ))); } + // Held until the fork commits, as the permissions save holds them: the source moved onto + // another database, or put under roles or taken off them, since its copy was made would link + // the copy to roles its grants were not replayed for. + let locked = crate::datatable_clone::lock_settings_rows(tx, parent_w_id, &fdt.name).await?; + let governing = ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?; + let under_roles = governing.datatable.permissions.is_some(); + let unchanged = crate::datatable_clone::same_database( + &governing.datatable.database, + &Some(copy.source_database.clone()), + ) && copy.replayed == under_roles + && crate::datatable_clone::lock_settings_rows(tx, parent_w_id, &fdt.name).await? == locked; + if !unchanged { + return Err(Error::BadRequest(format!( + "Data table '{}' changed while it was being copied for this fork — its database, or \ + whether it is under roles. Create the fork again.", + fdt.name + ))); + } + // The fork keeps a snapshot of the source's schema, which under roles is only for those who may + // connect as one of them — as the copy itself was. + crate::datatable_permissions::ensure_reaches_governing_datatable( + db, + parent_w_id, + &fdt.name, + &governing, + authed, + ) + .await?; + // Under roles, the copy holds rows the source's roles decide who reaches, so that entry keeps + // deciding. Settled from the source as it resolves now: the settings clone may have handed the + // fork a pointer, or a clone of its own. A copy of a data table without roles is the fork's, + // as it was before roles existed: everyone reached all of it already. + let governed_by = governing + .datatable + .permissions + .is_some() + .then(|| { + serde_json::to_value(governing.governor.clone().unwrap_or_else(|| { + windmill_common::workspaces::DataTableReference { + workspace_id: governing.workspace_id.clone(), + datatable: governing.name.clone(), + } + })) + }) + .transpose() + .map_err(|e| Error::internal_err(format!("serializing a clone's governor: {e}")))?; // Snapshot the schema from the source (parent) datatable let schema = snapshot_datatable_schema(db, parent_w_id, &fdt.name).await?; @@ -7214,33 +8739,63 @@ async fn apply_forked_datatable( let dt: DataTable = serde_json::from_value(config_val) .map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?; - if dt.database.resource_type == DataTableCatalogResourceType::Instance { - // Instance: update resource_path to the new dbname - sqlx::query!( + // A clone owns its copy, so the fork's entry has to be terminal. When the parent was itself a + // fork the settings clone hands down a pointer instead, and what it points at is what the copy + // was taken from. `ensure_datatable_is_clonable` already settled that this shape can be + // cloned, so there is nothing left to refuse here — by now the database exists and is filled. + let database = match dt.database.clone() { + Some(database) => database, + None => governing.datatable.database.clone().ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + fdt.name + )) + })?, + }; + + if database.resource_type == DataTableCatalogResourceType::Instance { + // The whole `database` object, not just its `resource_path`: a pointer entry has none to + // patch. `reference` goes with it — exactly one of the two may be set. + let new_database = serde_json::json!({ + "resource_type": "instance", + "resource_path": &fdt.new_dbname, + }); + sqlx::query( r#"UPDATE workspace_settings SET datatable = jsonb_set( - jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)), - ARRAY['datatables', $2, 'forked_from'], $4::jsonb - ) + datatable #- ARRAY['datatables', $2, 'reference'], + ARRAY['datatables', $2, 'database'], $3::jsonb) WHERE workspace_id = $1"#, - forked_w_id, - &fdt.name, - &fdt.new_dbname, - forked_from, ) + .bind(forked_w_id) + .bind(&fdt.name) + .bind(new_database) .execute(&mut **tx) .await?; } else { - // Resource: update the resource's dbname and mark as ws_specific - let resource_path = &dt.database.resource_path; - sqlx::query!( + // Resource: point it at the copy and mark it ws_specific. What the settings clone carried is + // the source's resource and variables as they are now, which an edit made while the copy + // was being made can have changed, and changed back: the clone has to be what the copy's + // connection was resolved from. + let resource_path = &database.resource_path; + let cloned = + crate::datatable_clone::connection_snapshot(db, &mut **tx, forked_w_id, resource_path) + .await?; + if copy.connection.as_ref() != Some(&cloned) { + return Err(Error::BadRequest(format!( + "The resource of data table '{}' changed while it was being copied for this \ + fork. Create the fork again.", + fdt.name + ))); + } + sqlx::query( r#"UPDATE resource SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text)) WHERE workspace_id = $1 AND path = $2"#, - forked_w_id, - resource_path, - &fdt.new_dbname, ) + .bind(forked_w_id) + .bind(resource_path) + .bind(&fdt.new_dbname) .execute(&mut **tx) .await?; @@ -7251,20 +8806,31 @@ async fn apply_forked_datatable( ) .execute(&mut **tx) .await?; - - // Set forked_from on the datatable config - sqlx::query!( - r#"UPDATE workspace_settings - SET datatable = jsonb_set(datatable, ARRAY['datatables', $2, 'forked_from'], $3::jsonb) - WHERE workspace_id = $1"#, - forked_w_id, - &fdt.name, - forked_from, - ) - .execute(&mut **tx) - .await?; } + // The settings clone copied the source's `permissions` along, and a clone of a clone its + // `governed_by`: a clone keeps no `permissions` of its own, and a link only under roles. + sqlx::query( + r#"UPDATE workspace_settings + SET datatable = jsonb_set( + CASE WHEN $3::jsonb IS NULL + THEN datatable #- ARRAY['datatables', $2, 'permissions'] + #- ARRAY['datatables', $2, 'governed_by'] + ELSE jsonb_set( + datatable #- ARRAY['datatables', $2, 'permissions'], + ARRAY['datatables', $2, 'governed_by'], $3::jsonb) + END, + ARRAY['datatables', $2, 'forked_from'], $4::jsonb + ) + WHERE workspace_id = $1"#, + ) + .bind(forked_w_id) + .bind(&fdt.name) + .bind(governed_by) + .bind(forked_from) + .execute(&mut **tx) + .await?; + Ok(()) } @@ -7314,6 +8880,76 @@ async fn enforce_cloud_fork_cap(db: &DB, parent_workspace_id: &str) -> Result<() enforce_cloud_fork_count(db, &root, 1).await } +/// Cloud: refuse to attach a workspace that already has a paid plan of its own. +/// +/// Once attached it draws the root's plan and meters its usage there, so a subscription of its own +/// bills a second time for one plan. Only an attach can reach this state: a fork is created as a +/// fresh workspace and never had a plan to keep. +/// +/// Asked only of a candidate joining this family, never of one already under the same root: that +/// one is already in the double-billed state, where the settings page surfaces the leftover +/// subscription and the portal that cancels it, and refusing there would block re-designating a +/// renamed dev workspace over a billing problem the attach did not cause. +#[cfg(feature = "cloud")] +async fn reject_attach_of_subscribed_workspace(db: &DB, dev_w_id: &str) -> Result<()> { + let plan = sqlx::query_scalar!( + "SELECT plan FROM workspace_settings WHERE workspace_id = $1", + dev_w_id + ) + .fetch_optional(db) + .await? + .flatten(); + // Any plan, not just `'team'`: the column is written by the subscription webhook, and a plan + // value it does not write yet would otherwise walk straight past this. An enterprise + // arrangement is deliberately not covered — it sets `premium` without a plan and has no + // self-serve portal, so refusing there would be a dead end rather than something to act on. + if plan.is_some() { + return Err(Error::BadRequest(format!( + "Workspace {dev_w_id} is on a paid plan of its own. A dev or fork workspace runs on its parent's plan and is never invoiced separately, so cancel that subscription from its own billing settings before attaching it." + ))); + } + Ok(()) +} + +#[cfg(all(test, feature = "cloud"))] +mod attach_billing_guard_tests { + use super::reject_attach_of_subscribed_workspace; + use sqlx::{Pool, Postgres}; + + async fn workspace_on_plan(db: &Pool, id: &str, plan: Option<&str>) { + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test-user')") + .bind(id) + .execute(db) + .await + .expect("insert workspace"); + sqlx::query("INSERT INTO workspace_settings (workspace_id, plan) VALUES ($1, $2)") + .bind(id) + .bind(plan) + .execute(db) + .await + .expect("insert workspace_settings"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn refuses_a_candidate_that_still_pays_for_itself(db: Pool) { + workspace_on_plan(&db, "subscribed", Some("team")).await; + workspace_on_plan(&db, "cancelled", None).await; + + let err = reject_attach_of_subscribed_workspace(&db, "subscribed") + .await + .expect_err("a workspace on a paid plan of its own must not be attachable"); + assert!(err.to_string().contains("paid plan of its own"), "{err}"); + + // Cancelling clears `plan` but keeps `customer_id`, so the plan column is what decides. + reject_attach_of_subscribed_workspace(&db, "cancelled") + .await + .expect("a workspace with no plan is attachable"); + reject_attach_of_subscribed_workspace(&db, "no-settings-row") + .await + .expect("a workspace with no settings row is attachable"); + } +} + /// General guardrail (all builds): reject creating a fork/dev under `parent` when it would nest deeper /// than `MAX_FORK_DEPTH`. `added_subtree_height` is the height of the subtree grafted below the new /// node — 0 for a plain fork, or the candidate's own subtree height for an attach. @@ -7529,7 +9165,169 @@ async fn create_workspace_fork( ensure_no_existing_dev_workspace(&db, &parent_workspace_id).await?; } + // Refused here, before any database exists; `make_copies` checks again under the locks. + validate_forked_datatables(&db, &authed, &parent_workspace_id, &nw).await?; + + // Detached from the request: a client that goes away while the copies are made must still end + // with the fork created or no copy left behind, and dropping the handler's future would skip + // that cleanup. + tokio::spawn(async move { + let fork_id = nw.id.clone(); + let copies = crate::datatable_clone::make_copies( + &db, + &authed, + &parent_workspace_id, + ©_requests(&nw.forked_datatables), + ) + .await?; + let replayed: Vec = copies + .iter() + .filter(|c| c.replayed) + .map(|c| c.behavior) + .collect(); + match write_workspace_fork( + db.clone(), + authed, + parent_workspace_id, + nw, + dev_workspace_label, + &copies, + ) + .await + { + Ok(message) => { + for behavior in replayed { + windmill_common::feature_usage::log_feature_usage( + "datatable", + "clone_replayed", + match behavior { + DataTableForkBehavior::SchemaOnly => "schema_only", + _ => "schema_and_data", + }, + ); + } + Ok(message) + } + // A commit whose acknowledgement was lost can still have committed, and a concurrent + // request for the same id can have: once the write has settled, the cleanup keeps + // whichever copies a committed fork names, and drops the rest. + Err(e) => match wait_for_fork_write(&db, &fork_id).await { + Ok(()) => Err(crate::datatable_clone::drop_copies_after(&db, copies, e).await), + Err(settle) => { + tracing::error!( + "Could not tell whether fork '{fork_id}' was created, so its copies were \ + kept: {settle}" + ); + Err(e) + } + }, + } + }) + .await + .map_err(|e| Error::internal_err(format!("Creating the fork stopped unexpectedly: {e}")))? +} + +/// The database a data table of the fork or dev workspace `fork_id` is copied into, as the wizard +/// and the CLI derive it. A dev workspace's id has no `wm-fork-` prefix, and its copies are dropped +/// by the same `wm_fork_` rule as a fork's. Dev workspace `x` and fork `wm-fork-x` share a name, as +/// their branches do: the second to copy a data table of that name is refused at `CREATE`. +fn forked_datatable_dbname(fork_id: &str, datatable: &str) -> String { + let suffix = fork_id + .strip_prefix(windmill_common::workspaces::WM_FORK_PREFIX) + .unwrap_or(fork_id); + format!("wm_fork_{}__{datatable}", suffix.replace('-', "_")) +} + +/// Wait until no transaction writing fork `fork_id` is still running: the lock +/// `write_workspace_fork` holds until its transaction ends is taken and released. +async fn wait_for_fork_write(db: &DB, fork_id: &str) -> Result<()> { + let mut tx = db.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork:' || $1))") + .bind(fork_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Everything about a fork's data tables that can be refused before a git branch or a database is +/// created, and that the fork request re-checks. +/// +/// Every data table is copied by the fork request, into a database it creates: an entry naming a +/// database that already exists could reach a copy made for another fork — one left behind by a +/// deleted fork included — without that copy's governance. +async fn validate_forked_datatables( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + nw: &CreateWorkspaceFork, +) -> Result<()> { + let mut names = HashSet::new(); + for fdt in &nw.forked_datatables { + if fdt.fork_behavior.is_none() { + return Err(Error::BadRequest(format!( + "Data table '{}' names no `fork_behavior`: the fork request makes the copy of each \ + data table itself. Update the Windmill CLI.", + fdt.name + ))); + } + let expected = forked_datatable_dbname(&nw.id, &fdt.name); + if fdt.new_dbname != expected { + return Err(Error::BadRequest(format!( + "Data table '{}' of fork '{}' is copied into database '{expected}', not '{}'", + fdt.name, nw.id, fdt.new_dbname + ))); + } + if !names.insert(fdt.name.as_str()) { + return Err(Error::BadRequest(format!( + "Data table '{}' is named more than once in this fork", + fdt.name + ))); + } + } + crate::datatable_clone::authorize_copies( + db, + authed, + parent_w_id, + ©_requests(&nw.forked_datatables), + ) + .await +} + +/// The copies a fork request asks this request to make. [`validate_forked_datatables`] refuses an +/// entry without a `fork_behavior`. +fn copy_requests( + forked_datatables: &[ForkedDatatableInfo], +) -> Vec> { + forked_datatables + .iter() + .filter_map(|fdt| { + fdt.fork_behavior + .map(|behavior| crate::datatable_clone::CopyRequest { + name: &fdt.name, + dbname: &fdt.new_dbname, + behavior, + }) + }) + .collect() +} + +/// Write the fork, with its data tables pointed at `copies`. +async fn write_workspace_fork( + db: DB, + authed: ApiAuthed, + parent_workspace_id: String, + nw: CreateWorkspaceFork, + dev_workspace_label: Option, + copies: &[crate::datatable_clone::MadeCopy], +) -> Result { let mut tx: Transaction<'_, Postgres> = db.begin().await?; + // Held until this transaction ends: after an error, the copies' cleanup waits on it, so it reads + // whether the fork exists only once a commit still being resolved has settled. + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork:' || $1))") + .bind(&nw.id) + .execute(&mut *tx) + .await?; if nw.is_dev_workspace { // The checks above ran outside a transaction, so the parent's eligibility and the chain's @@ -7579,8 +9377,19 @@ async fn create_workspace_fork( // intended. Dev creation is already admin-gated, so this is transitively admin-only too. Done before // the explicit creator insert below so the creator (a parent member) is copied with full metadata // (operator/role/is_service_account/added_via), not the bare row the insert alone would leave. + // Independently, the parent's admins can have every fork of it start with its admins and + // developers; the forker cannot opt out, since the point is that those admins can review it. if nw.copy_members && nw.is_dev_workspace { - copy_workspace_members(&mut tx, &parent_workspace_id, &forked_id).await?; + copy_workspace_members(&mut tx, &parent_workspace_id, &forked_id, false).await?; + } else if sqlx::query_scalar!( + "SELECT add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $1", + parent_workspace_id + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false) + { + copy_workspace_members(&mut tx, &parent_workspace_id, &forked_id, true).await?; } // Ensure the creator is a member of the fork even without copy_members (or if they aren't a parent @@ -7599,6 +9408,14 @@ async fn create_workspace_fork( .execute(&mut *tx) .await?; + // The pointers this fork writes to the parent's data tables stay invisible until it commits, so + // a rename of one of them cannot carry them. Holding the parent's settings row makes such a + // rename wait for this commit, and makes the copy below read one that committed first. + sqlx::query("SELECT 1 FROM workspace_settings WHERE workspace_id = $1 FOR SHARE") + .bind(&parent_workspace_id) + .execute(&mut *tx) + .await?; + // Clone all data from the parent workspace using Rust implementation if let Err(e) = clone_workspace_data(&mut tx, &db, &parent_workspace_id, &forked_id, &authed).await @@ -7634,11 +9451,53 @@ async fn create_workspace_fork( // re-enables in the fork, with parent-conflict warnings on enable. clone_triggers_and_schedules(&mut tx, &parent_workspace_id, &forked_id).await?; + repoint_unresolvable_cloned_identities(&mut tx, &forked_id, &authed).await?; + + // Held until the fork commits, as settings saves naming these databases hold it: an entry + // saved elsewhere while a copy was being made would reach it without the governance written + // below, and the alias check on that save saw no governed entry yet. + let instance_copies: Vec<&str> = copies + .iter() + .filter(|c| c.source_database.resource_type == DataTableCatalogResourceType::Instance) + .map(|c| c.dbname.as_str()) + .collect(); + windmill_common::lock_instance_databases(&mut tx, instance_copies.iter().copied()).await?; + for dbname in &instance_copies { + let users = windmill_common::instance_database_users(&mut *tx, dbname).await?; + if !users.is_empty() { + return Err(Error::BadRequest(format!( + "Database '{dbname}' copied for this fork is already used by workspaces {}; \ + fork again", + users.join(", ") + ))); + } + } + // Update forked datatable settings to point to new databases for fdt in &nw.forked_datatables { - apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; + let copy = copies.iter().find(|c| c.name == fdt.name).ok_or_else(|| { + Error::internal_err(format!("No copy was made of data table '{}'", fdt.name)) + })?; + apply_forked_datatable( + &db, + &mut tx, + &authed, + &parent_workspace_id, + &forked_id, + fdt, + copy, + ) + .await?; } + point_kept_datatables_at_parent( + &mut tx, + &parent_workspace_id, + &forked_id, + &nw.forked_datatables, + ) + .await?; + // The settings clone copies the source's ducklake config verbatim — including a parent // fork's own `fork_behavior` stamps. Sharing is a per-fork-creation choice, never // inherited: reset any cloned stamps first, then apply this fork's requested list. @@ -7687,6 +9546,20 @@ async fn create_workspace_fork( .await?; } + let copied = copies + .iter() + .map(|c| { + format!( + "{} ({})", + c.name, + match c.behavior { + DataTableForkBehavior::SchemaOnly => "schema_only", + _ => "schema_and_data", + } + ) + }) + .collect::>() + .join(", "); audit_log( &mut *tx, &authed, @@ -7694,7 +9567,7 @@ async fn create_workspace_fork( ActionKind::Create, &forked_id, Some(nw.name.as_str()), - None, + (!copied.is_empty()).then(|| [("copied_datatables", copied.as_str())].into()), ) .await?; tx.commit().await?; @@ -7856,6 +9729,17 @@ async fn attach_dev_workspace( ))); } + // Deliberately below the admin-of-candidate check, unlike the cap enforcement above: the + // refusal names the candidate's plan, so running it earlier would tell any admin of any + // premium workspace whether an arbitrary workspace id is on a team plan. + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + let root = windmill_common::workspaces::get_billing_workspace_id(&db, &prod_w_id).await?; + if windmill_common::workspaces::get_billing_workspace_id(&db, &dev_w_id).await? != root { + reject_attach_of_subscribed_workspace(&db, &dev_w_id).await?; + } + } + let mut tx = db.begin().await?; // Everything above ran outside a transaction, so prod's eligibility and the chain's labels could // have changed under us: re-decide both here, under the pairing lock. @@ -8430,6 +10314,14 @@ async fn leave_workspace( ) -> Result { windmill_api_auth::forbid_job_token_account_destruction(&authed)?; let mut tx = db.begin().await?; + // The membership is what made `u/` mean this person. Leaving it behind in a tenant + // list would hand their data table access back on rejoin, or to whoever takes the name next. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("u/{}", authed.username), + ) + .await?; sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND email = $2", &w_id, @@ -10913,6 +12805,7 @@ async fn load_workspace_authed( token_prefix: base_authed.token_prefix.clone(), read_only: base_authed.read_only, job_id: base_authed.job_id, + credential_expiry: base_authed.credential_expiry, }); }; @@ -10945,6 +12838,7 @@ async fn load_workspace_authed( token_prefix: base_authed.token_prefix.clone(), read_only: base_authed.read_only, job_id: base_authed.job_id, + credential_expiry: base_authed.credential_expiry, }) } diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index a8850a1144..48c49add1c 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -113,7 +113,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, variable_expiration_handler) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, variable_expiration_handler FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, variable_expiration_handler) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, variable_expiration_handler FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -187,6 +187,13 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + info!("Updating guest_activity table"); + sqlx::query("UPDATE guest_activity SET workspace_id = $1 WHERE workspace_id = $2") + .bind(&rw.new_id) + .bind(&old_id) + .execute(&mut *tx) + .await?; + info!("Updating workspace_invite table"); sqlx::query!( "UPDATE workspace_invite SET workspace_id = $1 WHERE workspace_id = $2", @@ -375,6 +382,13 @@ pub(crate) async fn change_workspace_id( ) .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE draft_move SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; info!("Updating favorite table"); sqlx::query!( @@ -485,6 +499,34 @@ pub(crate) async fn change_workspace_id( .fetch_all(&mut *tx) .await?; + // A fork's data table entry names the workspace that governs it by id, so the rename has to + // follow there too — anywhere, not just in the reparented children: a detached workspace can + // point at this one without being its fork. Left behind, the pointer resolves to the archived + // shell and every job through it stops. A clone's `governed_by` names it the same way. + info!("Re-pointing data table references to the new workspace id"); + sqlx::query!( + r#"UPDATE workspace_settings ws + SET datatable = ( + SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg( + dt.key, + (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $2 + THEN jsonb_set(r.v, '{governed_by,workspace_id}', to_jsonb($1::text)) + ELSE r.v END + FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $2 + THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text)) + ELSE dt.value END AS v) r) + )) + FROM jsonb_each(ws.datatable->'datatables') dt + ) + WHERE jsonb_typeof(ws.datatable->'datatables') = 'object' + AND (ws.datatable::text LIKE '%"reference"%' + OR ws.datatable::text LIKE '%"governed_by"%')"#, + &rw.new_id, + &old_id, + ) + .execute(&mut *tx) + .await?; + info!("Updating workspace_protection_rule table"); sqlx::query!( "UPDATE workspace_protection_rule SET workspace_id = $1 WHERE workspace_id = $2", @@ -964,6 +1006,24 @@ pub(crate) async fn delete_workspace( // but the destructive cleanup itself runs only after the commit below: a delete that // fails mid-way must never leave a live workspace with its fork data destroyed and no // registry row to retry from. Read-only: nothing is dropped here. + // Read before the delete: another workspace's data table entry can point at one of this + // workspace's, or be a clone taking its roles from one, and deleting the workspace it names + // leaves it resolving to nothing. Nothing sweeps them — turning them back into copies would + // hand each fork the database outright — so the deleter is told which data tables they just + // stranded. + let stranded_pointers = sqlx::query_as::<_, (String, String)>( + r#"SELECT ws.workspace_id, dt.key + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE dt.value->'reference'->>'workspace_id' = $1 + OR dt.value->'governed_by'->>'workspace_id' = $1 + ORDER BY ws.workspace_id, dt.key"#, + ) + .bind(&w_id) + .fetch_all(&db) + .await + .unwrap_or_default(); + let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None) .await .unwrap_or_else(|e| { @@ -1112,6 +1172,13 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + // Unlike the rest of this list, this also moves an instance-wide figure: the guest + // allowance and the seats past it are counted over every workspace's rows. + sqlx::query("DELETE FROM guest_activity WHERE workspace_id = $1") + .bind(&w_id) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM token WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; @@ -1275,7 +1342,23 @@ pub(crate) async fn delete_workspace( tracing::warn!("failed to broadcast fork lineage change: {e:#}"); } - Ok(format!("Deleted workspace {}", &w_id)) + if stranded_pointers.is_empty() { + Ok(format!("Deleted workspace {}", &w_id)) + } else { + let stranded = stranded_pointers + .iter() + .map(|(workspace_id, datatable)| format!("{workspace_id}/{datatable}")) + .collect::>() + .join(", "); + Ok(format!( + concat!( + "Deleted workspace {}. These data tables were governed by it and no longer ", + "resolve: {}. Their databases still exist; a superadmin can point them at ", + "another workspace's data table." + ), + &w_id, stranded + )) + } } #[derive(Deserialize)] @@ -1329,15 +1412,20 @@ pub async fn drop_forked_datatable_databases( let mut errors: Vec = Vec::new(); for dt_name in &req.datatable_names { - let dt = match datatables.get(dt_name) { - Some(dt) if dt.forked_from.is_some() => dt, + // Only a clone is droppable, and a clone is terminal by construction: a kept data table is + // a pointer at the parent's database, which this fork does not own. + let database = match datatables.get(dt_name) { + Some(dt) if dt.forked_from.is_some() => match dt.database.as_ref() { + Some(database) => database, + None => continue, + }, _ => continue, }; - if dt.database.resource_type + if database.resource_type == windmill_common::workspaces::DataTableCatalogResourceType::Instance { - let db_to_drop = &dt.database.resource_path; + let db_to_drop = &database.resource_path; if !db_to_drop.starts_with("wm_fork_") { errors.push(format!( "Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'", diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 99a6cd90f7..ad7f12453c 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private", "windmill-indexer?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-debug/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-amqp?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-api-npm-proxy/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] @@ -18,10 +18,12 @@ agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] -tantivy = ["dep:windmill-indexer"] +# The service log search handler reads the columnar store, so tantivy alone is +# not enough for this crate to build on its own. +tantivy = ["dep:windmill-indexer", "parquet"] kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 92eeaeaedb..3cba2b7da6 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -16532,202 +16532,6 @@ } } }, - "/w/{workspace}/ai_skills/list": { - "get": { - "summary": "list the workspace AI chat skills (name + description only)", - "operationId": "listAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "responses": { - "200": { - "description": "skill listing", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "required": [ - "name", - "description" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - } - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/get/{name}": { - "get": { - "summary": "get a workspace AI chat skill including its instructions", - "operationId": "getAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "skill", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "instructions": { - "type": "string" - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/upload": { - "post": { - "summary": "upsert workspace AI chat skills (admin only)", - "operationId": "uploadAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "skills" - ], - "properties": { - "skills": { - "type": "array", - "maxItems": 50, - "items": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9-]+$" - }, - "description": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "instructions": { - "type": "string", - "minLength": 1, - "maxLength": 65536 - } - } - } - } - } - } - } - } - }, - "responses": { - "200": { - "description": "uploaded", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/delete/{name}": { - "delete": { - "summary": "delete a workspace AI chat skill (admin only)", - "operationId": "deleteAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "deleted", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, "/w/{workspace}/apps/get_data/v/{secretWithExtension}": { "get": { "summary": "get raw app data by", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 235c80ccf9..d5a14a717d 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -17063,141 +17063,6 @@ paths: text/plain: schema: type: string - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - responses: - '200': - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: ^[a-z0-9-]+$ - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - '200': - description: uploaded - content: - text/plain: - schema: - type: string - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: deleted - content: - text/plain: - schema: - type: string /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 04f5f1d5ea..99265ff354 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.798.1 + version: 1.816.0 title: Windmill API contact: @@ -293,6 +293,108 @@ paths: items: $ref: "#/components/schemas/AuditLog" + /w/{workspace}/trash/list: + get: + summary: list the workspace trashbin (requires admin privilege) + operationId: listTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: item_kind + in: query + description: > + only return items of this kind: script, flow, app, schedule, variable, + resource, or a trigger kind such as http_trigger + schema: + type: string + - name: page + in: query + description: which page to return (starts at 0, default 0) + schema: + type: integer + - name: per_page + in: query + description: number of items to return for a given page (default 100, max 1000) + schema: + type: integer + responses: + "200": + description: the trashed items, most recently deleted first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TrashItem" + + /w/{workspace}/trash/get/{id}: + get: + summary: get a trashed item with the data it was deleted with (requires admin privilege) + operationId: getTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: the trashed item + content: + application/json: + schema: + $ref: "#/components/schemas/TrashItemWithData" + + /w/{workspace}/trash/restore/{id}: + post: + summary: restore a trashed item to its path (requires admin privilege) + operationId: restoreTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item restored + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/delete/{id}: + delete: + summary: permanently delete a trashed item (requires admin privilege) + operationId: permanentlyDeleteTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item permanently deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/empty: + post: + summary: permanently delete every item in the workspace trashbin (requires admin privilege) + operationId: emptyTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: trashbin emptied + content: + text/plain: + schema: + type: string + /auth/login: post: security: [] @@ -406,6 +508,28 @@ paths: "400": description: SMTP not configured + /auth/login_link/{token}: + get: + security: [] + summary: consume a single-use login link, set the session cookie and redirect + operationId: consumeLoginLink + tags: + - user + parameters: + - name: token + in: path + required: true + schema: + type: string + - name: rd + in: query + required: false + schema: + type: string + responses: + "302": + description: redirected to the post-login destination, or to /user/login_link_expired when the link is used, expired or unknown + /auth/reset_password: post: security: [] @@ -623,9 +747,14 @@ paths: skip_email: type: boolean description: Skip sending email notifications to the user + login_type: + type: string + description: >- + password (default, requires `password`), pending_oauth (no credential + until the first OAuth login proving the address adopts the account), or + a configured OAuth login client key required: - email - - password - super_admin responses: "201": @@ -861,6 +990,34 @@ paths: items: $ref: "#/components/schemas/ExternalJwtToken" + /users/guests: + get: + summary: list the distinct guests of the trailing window (superadmin only) + description: >- + The set the guest allowance is counted on: every distinct email that held a + guest session in the last `window_days`, with the workspaces it opened and the + days it was first and last seen, most recently seen first. `usage` is the + instance's standing against the allowance and the meter. + operationId: listGuests + tags: + - user + parameters: + - name: page + in: query + schema: + type: integer + - name: per_page + in: query + schema: + type: integer + responses: + "200": + description: the guests of the window and the allowance they count against + content: + application/json: + schema: + $ref: "#/components/schemas/GuestList" + /users/onboarding: post: summary: Submit user onboarding data @@ -1120,6 +1277,37 @@ paths: - archived - deleted + /workspaces/session_workspace_retention: + post: + summary: get the AI session retention of workspaces referenced by client-side sessions + operationId: getSessionWorkspaceRetention + tags: + - workspace + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + workspace_ids: + type: array + items: + type: string + required: + - workspace_ids + responses: + "200": + description: >- + map of workspace id to its `ai_config.sessions_retention_days`; a workspace + without a retention, or one the caller cannot be authenticated into, is absent + content: + application/json: + schema: + type: object + additionalProperties: + type: integer + /w/{workspace}/workspaces/get_as_superadmin: get: summary: get workspace as super admin (require to be super admin) @@ -1400,6 +1588,89 @@ paths: additionalProperties: $ref: "#/components/schemas/CustomInstanceDb" + /settings/datatable_roles: + get: + summary: list the instance's data table roles + operationId: listInstanceDatatableRoles + tags: + - setting + responses: + "200": + description: the instance role catalog + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/InstanceDatatableRole" + post: + summary: create a data table role on the instance's Postgres cluster + operationId: createInstanceDatatableRole + tags: + - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name] + properties: + name: + type: string + responses: + "200": + description: the created role + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceDatatableRole" + + /settings/datatable_roles/{id}: + post: + summary: rename a data table role or turn its login on and off + operationId: updateInstanceDatatableRole + tags: + - setting + parameters: + - in: path + name: id + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + enabled: + type: boolean + responses: + "200": + description: the updated role + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceDatatableRole" + delete: + summary: drop a data table role from the cluster and from every workspace that named it + operationId: deleteInstanceDatatableRole + tags: + - setting + parameters: + - in: path + name: id + required: true + schema: + type: string + responses: + "200": + description: deleted + /settings/setup_custom_instance_pg_database/{name}: post: summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user @@ -2713,7 +2984,7 @@ paths: /w/{workspace}/github_app/token: post: - summary: get github app token + summary: get the git credential for a git-sync job (GitHub App token, or the credential stored for the repository) operationId: getGithubAppToken tags: - workspace @@ -2733,7 +3004,7 @@ paths: - job_token responses: "200": - description: github app token + description: git credential content: application/json: schema: @@ -2877,6 +3148,131 @@ paths: "200": description: Successfully imported the installation + /w/{workspace}/git_sync/gitlab/projects: + post: + tags: + - Git Sync + summary: List the GitLab projects a token can sync + description: >- + Lists the projects the supplied GitLab token can push to, so a git + repository resource can be filled in without hand-writing a project + path. The token is used for this call only and is never stored. + Requires workspace admin. + operationId: listGitlabProjects + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + base_url: + type: string + description: The GitLab instance, e.g. https://gitlab.com + token: + type: string + description: A project access token with the api scope, or a group token that reaches the project + search: + type: string + description: Narrow the list to projects matching this text + required: + - base_url + - token + responses: + "200": + description: the projects the token can sync + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GitlabProject" + + /w/{workspace}/git_sync/credential/origin: + get: + tags: + - Git Sync + summary: Where a repository's credential comes from + description: >- + Whether Windmill holds this repository's access token, and which host it + talks to. `held` means this workspace stores it, `borrowed` means an + ancestor does and it is not this workspace's to replace. Both absent + means the repository authenticates with whatever its URL carries. + Returns no secret. Requires workspace admin. + operationId: getCredentialOrigin + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: >- + Path of the git repository resource, with or without the `$res:` + prefix. A path rather than a URL, because a resource URL may carry a + token and a URL in a query string lands in logs. + schema: + type: string + responses: + "200": + description: where the credential comes from + content: + application/json: + schema: + type: object + properties: + origin: + type: string + enum: + - held + - borrowed + provider: + type: string + enum: + - gitlab + + /w/{workspace}/git_sync/credential: + post: + tags: + - Git Sync + summary: Store the credential for a git repository + description: >- + Stores the access token a git repository authenticates with, so it does + not have to be written into the repository URL or a workspace variable. + The token is write-only: it is served only to a git-sync job that + presents its own job token, and a fork of this workspace reads this + copy instead of holding one of its own. Requires workspace admin. + operationId: setGitCredential + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + repo_url: + type: string + description: >- + The repository the credential is for, and the key it is + stored under. It is served for this repository and no other, + so repointing a resource elsewhere cannot carry the token + along. + token: + type: string + description: The access token, as pasted + required: + - repo_url + - token + responses: + "200": + description: the credential was stored + content: + text/plain: + schema: + type: string + /w/{workspace}/github_app/ghes_installation_callback: post: summary: GHES installation callback @@ -3667,8 +4063,16 @@ paths: $ref: "#/components/schemas/WorkspaceDeployUISettings" mute_critical_alerts: type: boolean + guest_access_enabled: + type: boolean + description: Whether this workspace admits guest sessions. An app's own `guest` execution mode is inert while this is false. + add_admins_and_developers_to_forks: + type: boolean + description: Whether every new fork of this workspace starts with its admins and developers as members, keeping their role. required: - workspace_id + - guest_access_enabled + - add_admins_and_developers_to_forks /w/{workspace}/workspaces/get_settings: get: @@ -3752,6 +4156,18 @@ paths: error_handler_fallback_to_instance_alerts: type: boolean description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. + guest_access_enabled: + type: boolean + description: Whether this workspace admits guest sessions. An app's own `guest` execution mode is inert while this is false. + guest_jwt_public_key: + type: string + description: PEM public key a guest JWT (`jwt_guest_`) is verified against for this workspace. Mutually exclusive with `guest_jwt_jwks_url`. + guest_jwt_jwks_url: + type: string + description: JWKS URL a guest JWT (`jwt_guest_`) is verified against for this workspace. Mutually exclusive with `guest_jwt_public_key`. + add_admins_and_developers_to_forks: + type: boolean + description: Whether every new fork of this workspace starts with its admins and developers as members, keeping their role. /w/{workspace}/workspaces/get_deploy_to: get: @@ -3790,6 +4206,38 @@ paths: schema: type: boolean + /w/{workspace}/workspaces/billable_seats: + get: + summary: get the billable seats of the workspace the plan is billed on + operationId: getBillableSeats + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + + responses: + "200": + description: billable seats + content: + application/json: + schema: + type: object + properties: + developers: + type: integer + description: >- + Omitted when the seats counted are another workspace's, as they are for a + fork resolving to its billing root. + operators: + type: integer + description: >- + Omitted when the seats counted are another workspace's, as they are for a + fork resolving to its billing root. + seats: + type: integer + required: + - seats + /w/{workspace}/workspaces/premium_info: get: summary: get premium info @@ -4842,7 +5290,7 @@ paths: type: array items: type: object - required: [name, resource_type, resource_path] + required: [name, resource_type, resource_path, permissioned] properties: name: type: string @@ -4851,6 +5299,188 @@ paths: enum: [postgres, instance] resource_path: type: string + governing_workspace_id: + type: string + permissioned: + type: boolean + + /w/{workspace}/workspaces/datatable_permissions/{datatable_name}: + get: + summary: get who may connect to a data table as which role + operationId: getDatatablePermissions + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: the data table's roles and their tenants + content: + application/json: + schema: + $ref: "#/components/schemas/DatatablePermissions" + post: + summary: set who may connect to a data table as which role + operationId: setDatatablePermissions + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [permissioned] + properties: + permissioned: + type: boolean + default_role: + type: string + roles: + type: array + items: + $ref: "#/components/schemas/DatatableRoleTenants" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}: + get: + summary: read the owner and grants of an instance data table's database, schema or table + operationId: getDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: kind + in: query + required: true + schema: + type: string + enum: [database, schema, table] + - name: schema + in: query + required: false + schema: + type: string + - name: table + in: query + required: false + schema: + type: string + responses: + "200": + description: owner and grants + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableAclInfo" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/plan: + post: + summary: preview the SQL an ownership or grant change would run (data table administrators only) + operationId: planDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: statements that would run, in a single transaction + content: + application/json: + schema: + $ref: "#/components/schemas/AclPlan" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/apply: + post: + summary: run an ownership or grant change exactly as planned (data table administrators only) + operationId: applyDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: change applied + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}: + get: + summary: list the data table roles the caller may connect as + operationId: listUsableDatatableRoles + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: usable roles + content: + application/json: + schema: + type: object + required: [permissioned, roles, default_role] + properties: + permissioned: + type: boolean + roles: + type: array + items: + type: string + default_role: + type: string /w/{workspace}/workspaces/list_datatable_schemas: get: @@ -4924,6 +5554,21 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: query + description: list only this data table; each listed data table opens a connection to its database + schema: + type: string + - name: role_for + in: query + description: the data table `role` applies to; every other one is listed as its default role + schema: + type: string + - name: role + in: query + description: the role to list `role_for` as; refused, in that entry's `error`, if the caller may not use it + schema: + type: string responses: "200": description: table metadata of all datatables @@ -4957,6 +5602,11 @@ paths: required: true schema: type: string + - name: role + in: query + description: the data table role to read the table as; defaults to the data table's default role + schema: + type: string responses: "200": description: schema of one datatable table @@ -5032,7 +5682,22 @@ paths: description: status content: application/json: - schema: {} + schema: + type: object + properties: + stranded_references: + description: >- + Data tables in other workspaces that were governed by one this save deleted + and no longer resolve. + type: array + items: + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string /w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}: post: @@ -5766,6 +6431,132 @@ paths: schema: type: string + /w/{workspace}/workspaces/edit_guest_access: + post: + summary: enable or disable guest sessions for this workspace + description: >- + Guests are people the identity provider authenticates who have no Windmill + account; the `guest` app execution mode admits them. Off by default. Re-read + where a guest session is minted and at the auth door on every guest request, so + turning it off takes effect immediately, for sessions already issued and for + apps whose policy already says `guest`. Turning it *on* is refused with a 400 + where guests are unavailable (the shared cloud); turning it off always works. + operationId: editGuestAccess + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Whether guest sessions are admitted + required: true + content: + application/json: + schema: + type: object + properties: + guest_access_enabled: + type: boolean + required: + - guest_access_enabled + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/edit_add_admins_and_developers_to_forks: + post: + summary: choose whether new forks of this workspace start with its admins and developers + description: >- + When on, every fork created from this workspace gets the workspace's admins and + developers as members, with the role they hold here; operators, disabled users and + service accounts are left out. The setting is copied into each fork, so forks of a + fork follow it too. Off by default. Workspace-admin gated. + operationId: editAddAdminsAndDevelopersToForks + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Whether new forks start with this workspace's admins and developers + required: true + content: + application/json: + schema: + type: object + properties: + add_admins_and_developers_to_forks: + type: boolean + required: + - add_admins_and_developers_to_forks + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/edit_guest_jwt_key: + post: + summary: set the key guest JWTs are verified against for this workspace + description: >- + A guest JWT (`jwt_guest_`) is minted by the embedding customer's own backend and + verified against this key: a PEM public key (RS/ES family, HS* refused) or a JWKS + URL, at most one. Both empty clears the workspace key; off cloud, verification then + falls back to the instance issuer (`JWT_EXT_JWKS_URL`) if one is set, else no guest + JWT is accepted (`guest_access_enabled` is the on/off switch). Workspace-admin gated. + The key is validated before it is stored. Setting a key is refused with a 400 where + guests are unavailable (the shared cloud); clearing one always works. + operationId: editGuestJwtKey + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: The guest JWT verification key + required: true + content: + application/json: + schema: + type: object + properties: + public_key: + type: string + description: A PEM public key (RS or ES family). + jwks_url: + type: string + description: A JWKS URL whose keys are fetched and refreshed. + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/guest_usage: + get: + summary: the instance's standing against the guest allowance + description: >- + Instance-wide, since a licence is per instance and one email is one guest however + many workspaces it opens. Read by workspace admins and app publishers to see how + close the cap (Community and Pro) or the meter (Enterprise) is. + operationId: getGuestUsage + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: guest usage + content: + application/json: + schema: + $ref: "#/components/schemas/GuestUsage" + /w/{workspace}/workspaces/default_scripts: post: summary: edit default scripts for workspace @@ -6152,6 +6943,172 @@ paths: schema: type: string + /users/login_links: + post: + summary: mint a single-use login link for an account (require superadmin) + operationId: createLoginLink + tags: + - user + requestBody: + description: target account and link options + required: true + content: + application/json: + schema: + type: object + required: + - email + properties: + email: + type: string + expires_in_s: + type: integer + description: link lifetime in seconds, at most 900 (default 600) + rd: + type: string + description: same-origin path the browser lands on after login (default /user/workspaces) + require_login_type: + type: string + description: >- + mint only while the account still has this login type (for example + pending_oauth), so a link stops working once the owner has set a password + or signed in with a provider + responses: + "201": + description: login link minted + content: + application/json: + schema: + type: object + required: + - url + - expires_at + properties: + url: + type: string + expires_at: + type: string + format: date-time + "409": + description: the account does not have the required login type + + /users/cloud_trial_offer: + post: + summary: record or consume a pre-approved self-hosted trial offer for a cloud account (require superadmin, cloud only) + operationId: setCloudTrialOffer + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - email + properties: + email: + type: string + consumed: + type: boolean + description: mark the offer used (a trial or subscription now exists) instead of recording it + responses: + "200": + description: offer recorded or consumed + content: + text/plain: + schema: + type: string + get: + summary: whether the signed-in account holds an unconsumed pre-approved self-hosted trial offer (cloud only) + operationId: getCloudTrialOffer + tags: + - user + responses: + "200": + description: offer state + content: + application/json: + schema: + type: object + required: + - offered + properties: + offered: + type: boolean + + /users/cloud_trial_offer/go: + post: + summary: start the signed-in account's pre-approved self-hosted trial on the customer portal (cloud only). A POST answered as JSON rather than a redirecting GET, so a cross-site navigation cannot start it; the browser navigates to `location` itself + operationId: goCloudTrialOffer + tags: + - user + responses: + "200": + description: where to go — the customer portal signed in with the trial being started, or the portal home with the reason the offer could not be used + content: + application/json: + schema: + type: object + required: + - location + properties: + location: + type: string + reason: + type: string + description: present when the portal refused (e.g. the account already has a subscription); the offer is then spent + "403": + description: called with a job token + "404": + description: no offer for this account + + /users/onboarding_profile: + post: + summary: record the invite context an account's onboarding tailors itself from (require superadmin, cloud only) + operationId: setOnboardingProfile + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - email + - profile + properties: + email: + type: string + profile: + type: object + description: free-form context from the invite, every key optional. The frontend reads `touch_point` (answers onboarding's source question), `company` and `workspace_name` (prefill the first workspace's name), `hub_projects` (slugs surfaced first on an empty workspace), `tools` (integrations, used to pick hub projects when none are named) and `starter_prompts` (`[{label, prompt}]`, replacing the home page's example prompts); unknown keys are kept and ignored + responses: + "200": + description: profile recorded + content: + text/plain: + schema: + type: string + get: + summary: the invite context recorded for the signed-in account, if any (cloud only) + operationId: getOnboardingProfile + tags: + - user + responses: + "200": + description: the profile, or null when none was recorded + content: + application/json: + schema: + type: object + properties: + profile: + type: object + nullable: true + additionalProperties: true + /users/tokens/delete/{token_prefix}: delete: summary: delete token @@ -8423,6 +9380,84 @@ paths: items: type: string + /w/{workspace}/resources/type/resource_counts: + get: + summary: count the workspace's resources per resource type + operationId: listResourceCountsByType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: resource count per resource_type + content: + application/json: + schema: + type: array + items: + type: object + properties: + resource_type: + type: string + count: + type: integer + required: + - resource_type + - count + + /w/{workspace}/resources/type/hub/info: + get: + summary: list what the hub knows about its resource types + operationId: listHubResourceTypeInfo + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: each hub resource type with its integration and pick count, empty if the hub answers neither read + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + app: + description: the integration the resource type belongs to, which is not always its own name + type: string + picks: + type: integer + required: + - name + - app + - picks + + /w/{workspace}/resources/type/hub/pick/{name}: + post: + summary: record a hub resource type pick + operationId: pickHubResourceType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Name" + responses: + "200": + description: whether the hub recorded the pick + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /w/{workspace}/npm_proxy/config: get: summary: get npm proxy configuration @@ -8672,6 +9707,13 @@ paths: properties: name: type: string + picks: + description: how often the integration has been picked, absent on a hub that does not count picks + type: integer + display_name: + description: the label the hub curates for the integration, null or absent where it names none + type: string + nullable: true required: - name @@ -8833,6 +9875,28 @@ paths: required: - app + /apps_u/guest_entry_by_custom_path/{custom_path}: + get: + summary: whether the app behind a custom path admits guests + description: >- + The custom-path counterpart of `getGuestEntry`. Unauthenticated; 404 unless + the app's execution mode is `guest` AND its workspace has + `guest_access_enabled` AND the instance has not set `guest_access_disabled`, + and never on a deployment where guests are unavailable (the shared cloud). + Returns the workspace too, since a custom URL may not carry it. + operationId: getGuestEntryByCustomPath + tags: + - app + parameters: + - $ref: "#/components/parameters/CustomPath" + responses: + "200": + description: the app is open to guests + content: + application/json: + schema: + $ref: "#/components/schemas/GuestEntry" + /apps_u/public_app_by_custom_path/{custom_path}: get: summary: get public app by custom path @@ -9468,8 +10532,49 @@ paths: current_timestamp: type: string format: date-time + path: + type: string + description: "`saved` only, upsert or delete: where the write landed. Differs from the URL path when the item had moved away from it; the editor follows it there. Absent when a delete found nothing to remove and the caller cannot read the path it moved to." required: [status, current_timestamp] + /w/{workspace}/drafts/move/{kind}/{path}: + post: + summary: move the current user's draft-only item to another path + description: Relocates the authed user's own draft row, along with both path keys inside its value (the typed path and the mirror the editors keep beside it). Only for draft-only items — a deployed item must be moved through its own deploy endpoint, which carries every draft with it. Restricted to script, flow, app and raw_app; any other kind is rejected with 400, because only these keep their deploy target where this endpoint rewrites it. + operationId: moveDraft + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + description: script, flow, app or raw_app only. + schema: + type: string + enum: [script, flow, app, raw_app] + - $ref: "#/components/parameters/ScriptPath" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + new_path: + type: string + summary: + type: string + description: Also restate the draft's summary. + required: [new_path] + responses: + "200": + description: move result + content: + text/plain: + schema: + type: string + /w/{workspace}/drafts/migrate_legacy/{kind}/{path}: post: summary: resolve a legacy (workspace-level) draft (admin only) @@ -9980,6 +11085,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: script history @@ -10483,7 +11590,7 @@ paths: summary: run script by path operationId: runScriptByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -10602,11 +11709,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid requestBody: description: script args @@ -10643,11 +11749,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid requestBody: description: script args @@ -10684,11 +11789,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid responses: "200": @@ -10711,11 +11815,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid - name: poll_delay_ms description: delay between polling for job updates in milliseconds in: query @@ -10753,11 +11856,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid - name: poll_delay_ms description: delay between polling for job updates in milliseconds in: query @@ -10793,11 +11895,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid - name: poll_delay_ms description: delay between polling for job updates in milliseconds in: query @@ -10841,11 +11942,10 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/SkipPreprocessor" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid - name: poll_delay_ms description: delay between polling for job updates in milliseconds in: query @@ -11345,6 +12445,13 @@ paths: properties: draft_only: type: boolean + chat_input_enabled: + type: boolean + description: | + `chat_input_enabled` of the flow's value, + projected so the list can mark flows that open + as a chat. Omitted when the value has no such + field. is_draft: type: boolean description: | @@ -11385,6 +12492,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" tags: - flow responses: @@ -11438,6 +12547,25 @@ paths: items: type: string + /w/{workspace}/flows/list_paths_linking_agent/{path}: + get: + summary: list flow paths with a step linked to a saved agent + operationId: listFlowPathsLinkingAgent + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: paths of the flows linking the `ai_agent` resource, as of their last deploy + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/flows/get/v/{version}: get: summary: get flow version @@ -11773,6 +12901,15 @@ paths: in: query schema: type: string + - name: kind + description: which conversations to list - the flow editor's test chats, the deployed flow's own (the default), or both + in: query + schema: + type: string + enum: + - test + - deployed + - all responses: "200": description: flow conversations list @@ -11783,6 +12920,40 @@ paths: items: $ref: "#/components/schemas/FlowConversation" + /w/{workspace}/flow_conversations/update/{conversation_id}: + post: + summary: rename flow conversation + operationId: updateFlowConversation + tags: + - flow_conversations + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: conversation_id + description: conversation id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [title] + properties: + title: + type: string + description: the chat's name + responses: + "200": + description: flow conversation updated + content: + text/plain: + schema: + type: string + /w/{workspace}/flow_conversations/delete/{conversation_id}: delete: summary: delete flow conversation @@ -12561,70 +13732,51 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports - /w/{workspace}/ai_skills/list: + /w/{workspace}/ai/sessions/list: get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills + summary: list the calling user's AI session backups in the workspace object storage + operationId: listAiSessionBackups tags: - - workspace + - ai parameters: - $ref: "#/components/parameters/WorkspaceId" responses: "200": - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: skill + description: backups, newest first; `enabled` is false when the workspace has no storage for them content: application/json: schema: type: object required: - - name - - description - - instructions + - enabled + - sessions properties: - name: - type: string - description: - type: string - instructions: + enabled: + type: boolean + storage_id: type: string + description: names the storage answered from; sync state recorded against another one is void + backup_generation: + type: integer + description: bumped by every workspace key rotation; sync state recorded under another one is void + fallback: + type: boolean + description: the storage answered from is the instance object store, standing in for a workspace without storage of its own; a removal owed to it is retired by any answer from the workspace's own storage + sessions: + type: array + description: the newest 500 at most + items: + $ref: "#/components/schemas/AISessionBackupListing" + truncated: + type: boolean + description: the user has more sessions than the answer names - /w/{workspace}/ai_skills/upload: + /w/{workspace}/ai/sessions/pull: post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills + summary: fetch whole AI session backups + operationId: pullAiSessionBackups tags: - - workspace + - ai parameters: - $ref: "#/components/parameters/WorkspaceId" requestBody: @@ -12634,55 +13786,229 @@ paths: schema: type: object required: - - skills + - ids properties: - skills: + ids: type: array - maxItems: 50 + maxItems: 20 items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: "^[a-z0-9-]+$" - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 + type: string + resume: + $ref: "#/components/schemas/AISessionBackupCursor" responses: "200": - description: uploaded + description: the backups found; `deferred` lists ids that did not fit the response budget content: - text/plain: + application/json: schema: - type: string + type: object + required: + - enabled + - sessions + - deferred + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + fallback: + type: boolean + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackup" + deferred: + type: array + items: + type: string - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill + /w/{workspace}/ai/sessions/push: + post: + summary: write changed pieces of AI sessions to their backups, and remove deleted ones + operationId: pushAiSessionBackups tags: - - workspace + - ai parameters: - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - owner + properties: + owner: + type: string + description: the email the push was prepared for; refused with a 409 when it is not the caller's + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackupPush" + removed: + type: array + items: + type: string + responses: + "200": + description: one result per session written or removed, in request order + content: + application/json: + schema: + type: object + required: + - enabled + - results + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + fallback: + type: boolean + results: + type: array + items: + type: object + required: + - id + properties: + id: + type: string + error: + type: string + needs_whole: + type: boolean + description: nothing was written and the session must be pushed whole again; an incremental part found no listed session to ride on (the backup was removed, or a push split over parts is in progress or was abandoned), or a later part of a push split over parts found another push had superseded it + + /w/{workspace}/ai/shared_artifacts/share: + post: + summary: share an AI session artifact with the workspace + description: > + Stores a read-only copy that any member of the workspace can open by id. Sharing the + same artifact again updates that copy, keeps its id, and restarts its retention window. + operationId: shareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - artifact_id + - name + - kind + - version + - content + properties: + artifact_id: + type: string + description: the artifact's id in the author's session + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + minimum: 1 + content: + type: string + responses: + "200": + description: the shared copy + content: + application/json: + schema: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/status: + get: + summary: get the calling user's share of one of their AI session artifacts + operationId: getAiArtifactShareStatus + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: artifact_id + in: query required: true schema: type: string responses: "200": - description: deleted + description: the live share, if any, and how long shares last + content: + application/json: + schema: + type: object + required: + - retention_secs + properties: + retention_secs: + type: integer + share: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/get/{id}: + get: + summary: get a shared AI session artifact + operationId: getSharedAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: the shared artifact + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/SharedAiArtifactInfo" + - type: object + required: + - content + - can_unshare + properties: + content: + type: string + can_unshare: + type: boolean + description: whether the caller authored the share or is a workspace admin + + /w/{workspace}/ai/shared_artifacts/delete/{id}: + delete: + summary: stop sharing an AI session artifact + operationId: unshareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: share deleted content: text/plain: schema: @@ -13046,6 +14372,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: app history @@ -13120,6 +14448,31 @@ paths: schema: type: string + /w/{workspace}/apps_u/guest_entry/{path}: + get: + summary: whether the app behind a share secret admits guests + description: >- + Unauthenticated: what a signed-out visitor reads to learn that signing in + would let them in. 404 unless the app's execution mode is `guest` AND the + workspace has `guest_access_enabled` AND the instance has not set the + `guest_access_disabled` global setting, and never on a deployment where guests + are unavailable (the shared cloud), so it says nothing about apps that are not + open to guests. Discloses only the app path, to a caller already + holding the share secret. + operationId: getGuestEntry + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: the app is open to guests + content: + application/json: + schema: + $ref: "#/components/schemas/GuestEntry" + /w/{workspace}/apps_u/public_app/{path}: get: summary: get public app by secret @@ -13288,9 +14641,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/create_raw_source: post: @@ -13430,9 +14783,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/update_raw/{path}: post: @@ -13486,9 +14839,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/custom_path_exists/{custom_path}: get: @@ -13968,7 +15321,7 @@ paths: summary: run flow by path operationId: runFlowByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -13997,11 +15350,10 @@ paths: schema: type: boolean - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid requestBody: description: flow args required: true @@ -14055,11 +15407,10 @@ paths: schema: type: boolean - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid requestBody: description: flow args required: true @@ -14547,11 +15898,10 @@ paths: type: boolean - $ref: "#/components/parameters/NewJobId" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid requestBody: description: preview @@ -14579,11 +15929,10 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - name: memory_id - description: memory ID for chat-enabled flows + description: Memory id for the flow's AI agent steps. A uuid is used as is; any other string is hashed within the workspace and flow, so the same string always names the same memory of that flow. in: query schema: type: string - format: uuid requestBody: description: preview @@ -16618,6 +17967,7 @@ paths: - can_approve - user_auth_required - approvers + - skin properties: flow_id: type: string @@ -16650,6 +18000,16 @@ paths: hide_cancel: type: boolean description: whether to hide the cancel button in the UI + skin: + type: string + enum: [detailed, minimal] + description: how the approval page presents the request + step_summary: + type: string + description: summary of the approval step, for the page title + flow_summary: + type: string + description: summary of the flow or workflow the approval belongs to approvers: type: array items: @@ -18723,6 +20083,49 @@ paths: schema: type: string + /w/{workspace}/native_triggers/{service_name}/setenabled/{external_id}: + post: + summary: set enabled state of native trigger + description: | + Enables or disables a native trigger. A disabled trigger stays registered on the + external service but starts no job when it fires. + Requires write access to the script or flow that the trigger is associated with. + operationId: setNativeTriggerEnabled + tags: + - native_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: service_name + in: path + required: true + schema: + $ref: "#/components/schemas/NativeServiceName" + - name: external_id + in: path + required: true + description: The external ID of the trigger from the external service + schema: + type: string + requestBody: + description: updated enabled state + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: native trigger enabled state updated + content: + text/plain: + schema: + type: string + /w/{workspace}/native_triggers/{service_name}/list: get: summary: list native triggers @@ -21558,6 +22961,98 @@ paths: - id - values + /workers/queue_metrics_series: + get: + summary: get the queue metrics of a time window, as a bounded line per tag + operationId: getQueueMetricsSeries + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 14-day retention) + schema: + type: integer + responses: + "200": + description: jobs waiting and queue delay per tag, as the vertices of lines joined by straight segments + content: + application/json: + schema: + type: object + properties: + from: + type: integer + description: start of the window, in epoch milliseconds + to: + type: integer + description: end of the window, in epoch milliseconds + tags: + type: array + items: + type: object + properties: + tag: + type: string + count: + type: array + description: "[epoch ms, jobs waiting more than 3 seconds] vertices" + items: + type: array + items: + type: number + delay: + type: array + description: "[epoch ms, seconds the next job has waited] vertices" + items: + type: array + items: + type: number + required: + - tag + - count + - delay + required: + - from + - to + - tags + + /workers/queue_status: + get: + summary: get the live queue status of every tag with jobs waiting or running + operationId: getQueueStatus + tags: + - worker + responses: + "200": + description: queue status per tag + content: + application/json: + schema: + type: array + items: + type: object + properties: + tag: + type: string + waiting: + type: integer + description: jobs due for more than 3 seconds that no worker has picked up + delay: + type: number + description: seconds the job the next pull would take has been waiting, absent when none is + running: + type: integer + workers: + type: integer + description: workers that pinged in the last minute and pull this tag + required: + - tag + - waiting + - running + - workers + /workers/queue_counts: get: summary: get counts of jobs waiting for an executor per tag @@ -23917,7 +25412,7 @@ paths: items: type: string hits: - description: log files that matched the query + description: the log lines that matched the query, newest first type: array items: $ref: "#/components/schemas/LogSearchHit" @@ -24341,6 +25836,72 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/assets/column_lineage: + get: + summary: Column-level lineage of a set of dbt relations + description: > + The direct (`copy` / `mod`) column-to-column lineage the given relations' + columns sit in — the connected component around them, from the engine's + static analysis. Not their own edges, which would stop one hop out since + a column trace walks transitively, and not a whole project's, which + carries model families the selection cannot reach. + + Several relations, answered as one union, because one selection reaches + several: a script's output column can derive from columns of several dbt + models. Unpinned, the component crosses projects — a relation one project + produces is another's source — and the caller's access is decided again + for every project it reaches, so a trace ends where their grants do. A + pinned answer, by version here or by job on the run route, is one + project's. + + Its own endpoint rather than a field on the asset graph: the graph is + folder-wide and polled by a run page, while this is rendered for one + selection at a time. Empty for projects that did not opt into the + analysis pass (`column_lineage: true`), which is the ordinary case. The + indirect `scan` kind is stored but never served: it reaches every output + column of its model. + operationId: getDbtColumnLineage + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. + schema: + type: array + maxItems: 1000 + minItems: 1 + items: + type: string + - name: dbt_script_hash + in: query + description: > + The deployed version a view is drawing, when it is drawing one — the + dbt editor, which shows a single project as of a single deploy. A + version-pinned answer is that version's project alone, the same as a + job-pinned one, and only the unpinned answer crosses projects: a pin + says which stored graph is on screen, and another project's live + graph is not part of it. + + A run's or an editor buffer's own graph is not reachable here: that + pins to a job, and costs the job-read gate — see + `jobs/dbt_column_lineage/{id}`. + schema: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/assets/macros: get: summary: List every workspace DuckDB macro (deployed `// macros` libraries) @@ -24532,6 +26093,53 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/jobs/dbt_column_lineage/{id}: + get: + summary: Get relations' project column lineage as one run saw it + description: > + The same answer as `assets/column_lineage`, for the project version a + single job ran — including the dbt editor's parse of its own buffer, + whose graph belongs to that job and is reachable no other way. One + project answers here, the one the run is of, since the graph this + annotates is that project's too. Authorized through the job, the same + gate as `dbt_graph`. Reaching the run is not on its own enough to read + the project: a caller with no access to the script gets its relations and + `ref()` edges from `dbt_graph` and an empty answer here, exactly as that + endpoint redacts the model's SQL. + operationId: getDbtRunColumnLineage + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: The job whose graph the lineage is read from + schema: + type: string + format: uuid + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. + schema: + type: array + maxItems: 1000 + minItems: 1 + items: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/jobs/run_progress/{id}: get: summary: List the per-relation progress one job has recorded so far @@ -25408,6 +27016,30 @@ paths: schema: type: string + /w/{workspace}/hub/projects: + get: + summary: list the hub's published projects + description: | + Forwards to the configured Hub's public project catalogue and returns its + status code and raw response body. Readable by any workspace member: the + listing is not workspace-scoped, and it is proxied only because the Hub's + listing endpoint sends no CORS header. Refused with 400 when the instance + has the Hub disabled, in which case no outbound request is made. + operationId: listHubProjects + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + "400": + description: the Hub is disabled on this instance + /w/{workspace}/hub/project: get: summary: get the hub project linked to a workspace folder @@ -26125,6 +27757,40 @@ components: drawn identically and the ambiguity would otherwise just move into the editor. Omitted for the unpinned workspace graph, which spans every project and so has no one time. + DbtColumnLineage: + type: object + description: >- + The direct column-to-column lineage the asked-for relations' columns sit + in — the connected component around them — in the terms the canvas draws: + relations and columns, never dbt's node ids. + required: [edges, truncated] + properties: + edges: + type: array + items: + type: object + required: [from_asset_path, from_column, to_asset_path, to_column, kind] + properties: + from_asset_path: + type: string + from_column: + type: string + to_asset_path: + type: string + to_column: + type: string + kind: + type: string + description: >- + dbt's own word for how the value travelled — `copy` + (passthrough) or `mod` (transformed). Not an enum: the engine + treats the set as open. + truncated: + type: boolean + description: >- + The component reaches further than `edges`, which holds the part + nearest the asked-for relations. A trace that stops short is + otherwise indistinguishable from one that ends. DbtAssetProvenance: type: object description: >- @@ -26172,7 +27838,23 @@ components: columns: type: object additionalProperties: true - description: Declared column metadata (name -> description). NOT column lineage — `manifest.json` carries none. + description: Declared column metadata (name -> description) — what `manifest.json` carries, which is only the columns an author wrote down. Omitted when the caller cannot read the script. + column_schema: + type: array + description: >- + Every column of the relation, typed and in the order the model + produces them, from the engine's static analysis. Present only for a + project that opted into it, and gated like `columns` and the model's + SQL: a full column list is the shape of what the author wrote. + items: + type: object + required: [name] + properties: + name: + type: string + type: + type: string + description: The declared type where `schema.yml` gives one, else the inferred one. Omitted when neither is known. freshness: type: object additionalProperties: true @@ -26207,6 +27889,13 @@ components: draft_saved_at: type: string format: date-time + draft_base: + type: string + description: | + The deployed version the draft forked from, as text whatever the + kind (script hash, flow version id, app version id). Compare to the + deployed head to tell a draft that is behind. Absent when there is + no draft or it was never forked from a deploy. no_deployed: type: boolean draft: @@ -27124,7 +28813,7 @@ components: FlowConversation: type: object required: - [id, workspace_id, flow_path, created_at, updated_at, created_by] + [id, workspace_id, flow_path, created_at, updated_at, created_by, is_test] properties: id: type: string @@ -27151,6 +28840,9 @@ components: created_by: type: string description: Username who created the conversation + is_test: + type: boolean + description: Started from the flow editor's test panel rather than a deployed run FlowConversationMessage: type: object @@ -27190,6 +28882,50 @@ components: success: type: boolean description: Whether the message is a success + tool_arguments: + type: string + nullable: true + description: >- + On a tool row, the arguments the model wrote for the call. For a script, flow or + AI agent tool these exclude the inputs its step wires in, which only the tool's + job holds. Null for a provider-native web search, whose query the provider does + not return. + tool_result: + type: string + nullable: true + description: >- + On a tool row, the text the model got back from the call, or what the call + failed with — the row's own text names the tool rather than the reason. For a + provider-native web search, its citations. + reasoning: + type: string + nullable: true + description: >- + On an answer, the thinking that produced it; on a tool row, the thinking that + led to the call. Each round's thinking is on one row. The agent job's result + keeps the turn's thinking as a single string. + attachments: + type: array + nullable: true + description: >- + The files a user message carried, as object-storage references: every flow + input other than user_message that held one or a list of them, at most 20. Never + file bytes or a presigned URL. + items: + type: object + required: [input, s3] + properties: + input: + type: string + description: The flow input that held the file + s3: + type: string + description: The file's key in object storage + storage: + type: string + description: The secondary storage holding the file, absent for the primary one + filename: + type: string EndpointTool: type: object @@ -27310,10 +29046,201 @@ components: type: integer minimum: 1 maximum: 2000000 + free_tier: + $ref: "#/components/schemas/FreeTierInfo" model_pricing: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + context_window_per_model: + type: object + description: >- + Context window in tokens per `provider:model`, overriding the built-in + table the AI chat uses to decide when to compact its history. + additionalProperties: + type: integer + minimum: 1024 + maximum: 10000000 + copilot_disabled: + type: boolean + description: >- + Hides the Windmill AI assistant (chat, sessions, code generation, completion, + fixes) from the workspace UI. Read from the workspace's own settings even when + the providers served fall back to the instance config. AI agent steps and the + AI sandbox in flows are unaffected. + sessions_storage_disabled: + type: boolean + description: >- + Stops browsers from backing their AI sessions up to the workspace's object + storage. Read from the workspace's own settings like `copilot_disabled`. + sessions_retention_days: + type: integer + minimum: 1 + maximum: 3650 + description: >- + The server deletes the backup of a session no push has reached for this many + days. Unset keeps backups until the user deletes the session. Read from the + workspace's own settings like `copilot_disabled`. + + AISessionBackupListing: + type: object + required: + - id + - updated_at + - epoch + properties: + id: + type: string + updated_at: + type: string + format: date-time + epoch: + type: integer + description: the session's move count when this copy was pushed; of a session two workspaces list, the copy with the higher one is the later + + AISessionBackupImage: + type: object + required: + - chat_id + - id + - data_url + properties: + chat_id: + type: string + id: + type: string + data_url: + type: string + + AISessionBackupChat: + type: object + required: + - id + - record + properties: + id: + type: string + record: + type: object + additionalProperties: true + + AISessionBackup: + type: object + required: + - id + - head + - chats + - images + - listing + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + next: + $ref: "#/components/schemas/AISessionBackupCursor" + listing: + type: string + description: a fingerprint of the session's listing; pages of one session whose fingerprints differ do not belong together + moved: + type: boolean + description: the backup kept changing while this page was read, so it may mix two versions; the browser starts the session over + + AISessionBackupCursor: + type: object + description: where a pull of a session that did not fit one answer whole picks up; the rest of the session follows a pull naming that session alone with this as `resume` + required: + - id + - images + - after + properties: + id: + type: string + images: + type: boolean + after: + type: string + + AISessionBackupPush: + type: object + required: + - id + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + delete_chats: + type: array + items: + type: string + delete_images: + type: array + items: + type: object + required: + - chat_id + - id + properties: + chat_id: + type: string + id: + type: string + partial: + type: boolean + description: more parts of this session follow, in this push or a later one; the session is not listed on this one. Such a part names its push (`push`), or it is refused + whole: + type: boolean + description: a part of a push of the session whole; the head is on the part that opens it, which replaces whatever the storage holds of the session, and every piece the browser has is on one of them. An incremental part instead rides on a session the storage lists and is refused with needs_whole when it lists none + push: + type: string + description: a push split over several parts names itself on each with a token the browser draws; the part that opens it unlists the session and the last part lists it again, and a later part is written only while that token is the one there (refused with needs_whole otherwise) + opens: + type: boolean + description: this part opens the push named by `push` + epoch: + type: integer + description: the session's move count (its record's `moves`), kept with the marker that lists the session; an incremental part rides on the marker of the same count + + FreeTierInfo: + type: object + description: >- + Read-only. Present when the workspace has no AI provider of its own and is running + on Windmill's free tier. Ignored on write. + properties: + exhausted: + type: boolean + description: The one-time grant is spent; no provider is served and the user must add their own API key. + used_ratio: + type: number + description: Fraction of the grant consumed, 0 to 1. + required: + - exhausted + - used_ratio ModelPriceOverride: type: object @@ -27397,6 +29324,36 @@ components: - output_tokens - requests + SharedAiArtifactInfo: + type: object + properties: + id: + type: string + format: uuid + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + created_by: + type: string + shared_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + required: + - id + - name + - kind + - version + - created_by + - shared_at + - expires_at + InstanceAIProviderSummary: type: object properties: @@ -27569,6 +29526,12 @@ components: type: boolean has_deploy_errors: type: boolean + chat_input_enabled: + type: boolean + description: >- + flow-only. `chat_input_enabled` of the flow's value, projected so + the list can mark flows that open as a chat. Omitted when the + value has no such field. raw_app: type: boolean execution_mode: @@ -27864,6 +29827,11 @@ components: type: string deployment_msg: type: string + created_at: + type: string + format: date-time + created_by: + type: string required: - script_hash @@ -28076,7 +30044,10 @@ components: type: string args: $ref: "#/components/schemas/ScriptArgs" - result: {} + result: + description: | + For large results, this may be the placeholder string 'WINDMILL_TOO_BIG'. + Use the completed job result endpoint to retrieve the full result. logs: type: string deleted: @@ -28880,6 +30851,82 @@ components: - is_operator - last_used_at + GuestUsage: + type: object + description: >- + Guests are free up to `free_allowance` distinct emails over the trailing + `window_days`. Past that an Enterprise plan meters them (`metered`, four guests + to one seat: `billable_guests`, `guest_seats`); every other plan and build + admits no new email until the count drops. `instance_enabled` is the superadmin + switch (`guest_access_disabled` global setting) every workspace switch sits under. + `available` is whether this deployment can have guests at all: false on the shared + cloud, where guest access requires a self-hosted or dedicated deployment, and every + other field and switch is then moot. + properties: + available: + type: boolean + instance_enabled: + type: boolean + guest_count: + type: integer + format: int64 + window_days: + type: integer + free_allowance: + type: integer + format: int64 + metered: + type: boolean + billable_guests: + type: integer + format: int64 + guest_seats: + type: integer + format: int64 + required: + - available + - instance_enabled + - guest_count + - window_days + - free_allowance + - metered + - billable_guests + - guest_seats + + GuestActivity: + type: object + properties: + email: + type: string + workspaces: + type: array + items: + type: string + first_seen: + type: string + format: date + last_seen: + type: string + format: date + required: + - email + - workspaces + - first_seen + - last_seen + + GuestList: + type: object + properties: + usage: + $ref: "#/components/schemas/GuestUsage" + guests: + type: array + items: + $ref: "#/components/schemas/GuestActivity" + required: + - usage + - guests + NewToken: type: object properties: @@ -29169,6 +31216,8 @@ components: - "igroup.delete" - "igroup.adduser" - "igroup.removeuser" + - "instance_groups.jit_adduser" + - "instance_groups.jit_removeuser" - "variables.decrypt_secret" - "workspaces.read_encryption_key" - "workspaces.edit_command_script" @@ -29200,6 +31249,51 @@ components: - operation - action_kind + TrashItem: + type: object + properties: + id: + type: integer + format: int64 + workspace_id: + type: string + item_kind: + type: string + description: script, flow, app, schedule, variable, resource, or a trigger kind such as http_trigger + item_path: + type: string + deleted_by: + type: string + deleted_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + description: when the item is permanently deleted unless restored first + required: + - id + - workspace_id + - item_kind + - item_path + - deleted_by + - deleted_at + - expires_at + + TrashItemWithData: + allOf: + - $ref: "#/components/schemas/TrashItem" + - type: object + properties: + item_data: + type: object + additionalProperties: true + description: > + the deleted rows as they were stored; the shape depends on the kind, and a + secret variable's value stays encrypted + required: + - item_data + MainArgSignature: type: object properties: @@ -29657,6 +31751,11 @@ components: type: string is_fileset: type: boolean + display_name: + type: string + description: >- + The name the product goes by, e.g. "Google Sheets" for gsheets. + Absent where nobody named the type. required: - name @@ -29668,6 +31767,18 @@ components: type: string is_fileset: type: boolean + format_extension: + type: string + nullable: true + description: >- + File extension for a type whose value is one file rather than a set + of fields. Omit to leave it unchanged; send null to clear it. + display_name: + type: string + nullable: true + description: >- + The name the product goes by. Omit to leave it unchanged; send null + to clear it. TriggerHistoryEntry: type: object @@ -30371,9 +32482,19 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30460,9 +32581,19 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30556,9 +32687,19 @@ components: raw_string: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + nullable: true + maxItems: 100 + items: + type: string + maxLength: 256 + description: "Origins allowed to call this route cross-origin, matched against the request's Origin header (ignoring case) and echoed back on a match. When set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin the runnable returns via wm_headers. Use ['*'] to opt out of any restriction, including the http_route_default_allowed_origins instance setting. An empty list is not a configuration and resolves exactly as null does. When null, the instance setting applies, or Access-Control-Allow-Origin: * if it is unset. Ignored on a static website, which has no authentication of its own and so hands out public files: restricting which browsers may read them protects nothing while breaking cross-origin webfonts and fetches. A single-file static asset is not exempt, since it can carry an authentication_method." error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30750,7 +32891,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30815,7 +32958,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30891,7 +33036,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -31019,7 +33166,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31070,7 +33219,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31134,7 +33285,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31217,7 +33370,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31258,7 +33413,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31312,7 +33469,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31710,7 +33869,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31749,6 +33910,8 @@ components: $ref: "#/components/schemas/LoggedWizardStatus" replication_user_error: type: string + user_connect: + $ref: "#/components/schemas/LoggedWizardStatus" CustomInstanceDbTag: type: string @@ -31756,6 +33919,327 @@ components: - ducklake - datatable + InstanceDatatableRole: + type: object + required: [id, name, enabled] + properties: + id: + type: string + name: + type: string + enabled: + type: boolean + + DatatableRoleTenants: + type: object + required: [id, tenants] + properties: + id: + type: string + name: + type: string + tenants: + type: array + items: + type: string + + DatatablePermissions: + type: object + required: [supported, permissioned, default_role, roles, editable, available_roles] + properties: + supported: + type: boolean + description: >- + Whether this data table can be put under roles at all. Only one backed by the + instance database can: a role is a login on that cluster. + permissioned: + type: boolean + default_role: + type: string + roles: + type: array + items: + $ref: "#/components/schemas/DatatableRoleTenants" + governing_workspace_id: + type: string + clone_of: + type: object + description: for a clone, the data table whose roles it takes + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string + editable: + type: boolean + available_roles: + type: array + items: + $ref: "#/components/schemas/InstanceDatatableRole" + ungoverned_reachers: + type: array + items: + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string + + AclTarget: + description: what access is read or changed on + oneOf: + - $ref: "#/components/schemas/AclTargetDatabase" + - $ref: "#/components/schemas/AclTargetSchema" + - $ref: "#/components/schemas/AclTargetTable" + discriminator: + propertyName: kind + mapping: + database: "#/components/schemas/AclTargetDatabase" + schema: "#/components/schemas/AclTargetSchema" + table: "#/components/schemas/AclTargetTable" + + AclTargetDatabase: + type: object + required: [kind] + properties: + kind: + type: string + enum: [database] + + AclTargetSchema: + type: object + required: [kind, schema] + properties: + kind: + type: string + enum: [schema] + schema: + type: string + + AclTargetTable: + type: object + required: [kind, schema, table] + properties: + kind: + type: string + enum: [table] + schema: + type: string + table: + type: string + + AclChange: + description: one change to plan or apply + oneOf: + - $ref: "#/components/schemas/AclChangeSetOwner" + - $ref: "#/components/schemas/AclChangeGrant" + - $ref: "#/components/schemas/AclChangeRevoke" + discriminator: + propertyName: type + mapping: + set_owner: "#/components/schemas/AclChangeSetOwner" + grant: "#/components/schemas/AclChangeGrant" + revoke: "#/components/schemas/AclChangeRevoke" + + AclChangeSetOwner: + type: object + description: >- + hands the target to role — for a schema, with everything already in it but an extension's + members, which stay with the extension + required: [type, role] + properties: + type: + type: string + enum: [set_owner] + role: + type: string + description: a data table role of the instance, or admin + + AclChangeGrant: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [grant] + role: + type: string + description: a data table role of the instance, or admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + + AclChangeRevoke: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [revoke] + role: + type: string + description: a data table role of the instance, other than admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + objects: + type: array + description: >- + objects inside the target the revoke covers, empty for the target itself. Only with the + target scope; a revoke on all objects of a kind is refused, since it cannot say which + grants it takes back. + items: + $ref: "#/components/schemas/AclObject" + + AclGrantScope: + type: string + enum: + [ + target, + all_tables, + all_sequences, + all_functions, + future_tables, + future_sequences, + future_functions, + ] + + AclChangeRequest: + type: object + required: [target, change] + properties: + target: + $ref: "#/components/schemas/AclTarget" + change: + $ref: "#/components/schemas/AclChange" + statements: + type: array + description: >- + The statements the plan showed. Required to apply, which plans again and refuses if + the result differs. + items: + type: string + + AclPlan: + type: object + required: [statements, warnings] + properties: + statements: + type: array + items: + type: string + warnings: + type: array + items: + type: string + + AclObject: + type: object + required: [name, kind] + properties: + name: + type: string + kind: + type: string + description: >- + TABLE, SEQUENCE, FUNCTION, PROCEDURE or TYPE — what the object is. A revoke turns it + into the keyword it takes, ROUTINE for both routine kinds; a type's grants are read + only. + args: + type: string + description: identity arguments of a routine, which is what tells two of the same name apart + + AclGrant: + type: object + required: [grantee, privileges, sources] + properties: + grantee: + type: string + privileges: + type: array + items: + type: string + object: + $ref: "#/components/schemas/AclObject" + future: + type: string + description: >- + set for a default privilege, naming the kind of object it covers (TABLES, SEQUENCES, + FUNCTIONS, TYPES, or SCHEMAS). On a schema, the defaults set in that schema; on the + database, the ones set database-wide, which apply in every schema and which no schema's + own defaults take back. + sources: + type: array + description: >- + the roles the grant comes from, each once — who granted it, or for a default privilege + the role whose future objects it covers. A revoke of some of the grant's privileges + takes them back from every source that gave them. + items: + $ref: "#/components/schemas/AclSource" + + AclSource: + type: object + required: [role, privileges, reachable] + properties: + role: + type: string + privileges: + type: array + description: >- + what role gave of the grant's privileges. A revoke is held back only by a source out of + reach that gave some of what it takes back. + items: + type: string + reachable: + type: boolean + description: >- + whether the data table's connection can take back what role gave. On an object that is + the owner, when the connection acts for the owner, and otherwise the connection itself; + for a default privilege, a creating role the connection acts for. What a source out of + reach gave is not revocable from here; privileges only other sources gave still are. + + DatatableAclInfo: + type: object + required: [owner, roles, editable, clone, supports_maintain, dbname, grants, children] + properties: + owner: + type: string + roles: + type: array + description: the roles a change may name; empty unless the caller may change anything + items: + type: string + editable: + type: boolean + description: whether the caller may plan and apply changes + clone: + type: boolean + description: whether this is a clone, whose grants stay as they were copied + supports_maintain: + type: boolean + description: whether the server is Postgres 17+, which added the MAINTAIN table privilege + dbname: + type: string + description: the database the target lives in + grants: + type: array + items: + $ref: "#/components/schemas/AclGrant" + children: + type: array + description: a database's schemas, or a schema's tables + items: + type: string + CustomInstanceDb: type: object required: @@ -31812,7 +34296,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31868,7 +34354,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31996,7 +34484,9 @@ components: description: Timestamp of last server heartbeat (internal) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32036,7 +34526,9 @@ components: description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32088,7 +34580,9 @@ components: description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32165,7 +34659,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32229,7 +34725,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32304,7 +34802,9 @@ components: description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32367,7 +34867,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32415,7 +34917,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32474,7 +34978,9 @@ components: description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32816,6 +35322,9 @@ components: nullable: true disabled: type: boolean + is_service_account: + type: boolean + description: Whether this membership is a service account. required: - id - name @@ -32867,6 +35376,16 @@ components: new_dbname: type: string description: "New database name for the fork" + fork_behavior: + type: string + enum: + - schema_only + - schema_and_data + description: >- + What the fork request copies into `new_dbname`, which it creates — with the + owners and grants of a data table under roles. This server refuses an entry + without it; servers predating it expect `new_dbname` created and filled + beforehand. shared_ducklakes: type: array items: @@ -33002,7 +35521,7 @@ components: type: string login_type: type: string - enum: ["password", "github", "service_account"] + enum: ["password", "github", "service_account", "pending_oauth"] super_admin: type: boolean devops: @@ -33257,18 +35776,23 @@ components: type: string execution_mode: type: string - enum: [viewer, publisher, anonymous] + enum: [viewer, publisher, guest, anonymous] description: >- - Who the app's runnables execute as. Optional, and what omitting it - means depends on the operation: creating an app defaults it to - `publisher` (runs on behalf of the app's publisher and requires an - authenticated viewer), while updating one keeps the mode the app is - already deployed under. Either way `anonymous`, which makes the app - publicly executable, is never assumed + Who may open the app, and who its runnables execute as. Optional, and + what omitting it means depends on the operation: creating an app + defaults it to `publisher` (runs on behalf of the app's publisher and + requires an authenticated viewer), while updating one keeps the mode + the app is already deployed under. Neither `anonymous`, which makes + the app publicly executable, nor `guest`, which opens it to anyone the + identity provider authenticates, is ever assumed. A guest is only + admitted where the workspace also has `guest_access_enabled`, which is + checked when the session is minted and again on every guest request on_behalf_of: type: string + description: The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity. on_behalf_of_email: type: string + description: Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected. sandbox: type: boolean description: > @@ -33289,7 +35813,8 @@ components: carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, - users:read, resources:read, variables:read). + users:read, resources:read, variables:read, flow_conversations:read, + flow_conversations:write). ListableApp: type: object @@ -33315,7 +35840,7 @@ components: format: date-time execution_mode: type: string - enum: [viewer, publisher, anonymous] + enum: [viewer, publisher, guest, anonymous] raw_app: type: boolean labels: @@ -33499,6 +36024,20 @@ components: - raw_app + AppDeployed: + type: object + description: "What a deploy of an existing app answers with. `version` is the one this call wrote, which is what an editor pins as the fork base of the draft it starts next: reading the head back afterwards cannot tell it from a deploy that landed beside it. A metadata-only update writes none and reports the head it kept." + properties: + path: + type: string + description: Where the app now lives, which differs from the request path on a rename. + version: + type: integer + format: int64 + required: + - path + - version + AppHistory: type: object properties: @@ -33506,6 +36045,11 @@ components: type: integer deployment_msg: type: string + created_at: + type: string + format: date-time + created_by: + type: string required: - version @@ -33575,6 +36119,8 @@ components: format: date-time deployment_msg: type: string + created_by: + type: string required: - id - created_at @@ -33776,9 +36322,11 @@ components: type: object additionalProperties: type: object - required: [database] properties: database: + description: >- + Set on an entry that owns its database. Absent on a fork's entry, which points at + another workspace's data table instead. type: object properties: resource_type: @@ -33790,6 +36338,28 @@ components: type: string required: - resource_type + reference: + description: >- + The workspace and data table that govern this one. Server-owned: written by fork + creation, and carried across a settings save whatever the request says. + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string + governed_by: + description: >- + On a clone, the data table it was copied from, whose roles it takes. Server-owned + like `reference`. + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string migrations_enabled: type: boolean description: Whether the SQL migrations feature is opted in for this data table @@ -33858,7 +36428,17 @@ components: DataTableTables: type: object - required: [datatable_name, schemas] + required: + [ + datatable_name, + schemas, + instance, + permissioned, + usable_roles, + default_role, + can_create_schema, + creatable_schemas, + ] properties: datatable_name: type: string @@ -33871,6 +36451,26 @@ components: type: string error: type: string + instance: + type: boolean + description: on the instance database, the only kind that can be under roles or have its access edited + permissioned: + type: boolean + usable_roles: + type: array + description: the roles the caller may connect as, by name; empty when not under roles + items: + type: string + default_role: + type: string + can_create_schema: + type: boolean + description: whether the role the listing connected as may create schemas + creatable_schemas: + type: array + description: the schemas the role the listing connected as may create in + items: + type: string DataTableTableSchema: type: object @@ -34110,9 +36710,58 @@ components: open_pr_error: type: string description: server-owned, last failure opening a PR for a deploy branch of this repo + credential: + $ref: "#/components/schemas/GitCredentialStatus" required: - git_repo_resource_path + GitCredentialStatus: + type: object + description: server-owned, what the repo's own credential reports about itself + properties: + provider: + type: string + enum: + - gitlab + token_id: + type: integer + format: int64 + expires_at: + type: string + format: date + description: absent for a non-expiring token + scopes: + type: array + items: + type: string + rotatable: + type: boolean + description: whether this workspace renews the credential itself + checked_at: + type: integer + format: int64 + error: + type: string + required: + - provider + - rotatable + - checked_at + + GitlabProject: + type: object + description: a GitLab project a token can sync, as the resource form needs it + properties: + path_with_namespace: + type: string + description: nested group path plus project name, which is also GitLab's project id + http_url_to_repo: + type: string + default_branch: + type: string + required: + - path_with_namespace + - http_url_to_repo + AutoPullMode: type: string enum: @@ -34165,6 +36814,9 @@ components: type: string last_pull_status: $ref: "#/components/schemas/AutoPullStatus" + enabled_by: + type: string + description: Email of the admin automatic pulls apply changes as. Set by the server when the settings are saved. required: - enabled @@ -34357,8 +37009,34 @@ components: LogSearchHit: type: object properties: - dancer: + ts: + description: timestamp of the log line itself, not of the file containing it type: string + format: date-time + host: + type: string + level: + type: string + enum: [TRACE, DEBUG, INFO, WARN, ERROR] + target: + description: the tracing target that emitted the line + type: string + nullable: true + message: + type: string + file_path: + description: the log file the line came from + type: string + line_no: + description: offset of the line within its file + type: integer + required: + - ts + - host + - level + - message + - file_path + - line_no AutoscalingEvent: type: object @@ -35103,6 +37781,17 @@ components: description: Configuration of protection restrictions items: $ref: "#/components/schemas/ProtectionRuleKind" + GuestEntry: + type: object + description: What a signed-out visitor needs to start a guest sign-in. + properties: + workspace_id: + type: string + app_path: + type: string + required: + - workspace_id + - app_path ProtectionRuleKind: type: string enum: @@ -35111,6 +37800,7 @@ components: - RestrictDeployToDeployers - RestrictAnonymousAppDeployment - RestrictPublicRunSharing + - RestrictGuestAppDeployment RuleBypasserGroups: type: array description: Groups that can bypass this ruleset @@ -35262,6 +37952,9 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: Whether the trigger starts a job when it fires required: - external_id - workspace_id @@ -35269,6 +37962,7 @@ components: - script_path - is_flow - service_config + - enabled NativeTriggerWithExternal: type: object @@ -35300,6 +37994,9 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: Whether the trigger starts a job when it fires external_data: type: object nullable: true @@ -35320,6 +38017,7 @@ components: - script_path - is_flow - service_config + - enabled - external_data WorkspaceIntegrations: @@ -35405,6 +38103,12 @@ components: type: string nullable: true description: Short summary to be displayed when listed + enabled: + type: boolean + description: >- + Whether the trigger starts a job when it fires. Honoured on create only, so a + trigger can be registered already paused; an update ignores it and setenabled is + the only way to change an existing trigger's state. Defaults to true. required: - script_path - is_flow diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1a5f9dba87..16b4b17141 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -23,7 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision; use windmill_ai::ai_providers::{ empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; -use windmill_ai::ai_types::MAX_MODEL_RATE; +use windmill_ai::ai_types::{validate_token_limit, CONTEXT_WINDOWS, MAX_MODEL_RATE, OUTPUT_LIMITS}; use windmill_ai::credentials::ProviderCredentials; #[cfg(feature = "bedrock")] use windmill_ai::providers::bedrock::{ @@ -409,6 +409,19 @@ impl ExpiringProviderCredentials { } } +/// Set on the copilot config when the workspace has no AI provider of its own and is +/// running on Windmill's free tier, so the client can label the lent model as free, warn +/// before the grant runs out, and tell the user to add their own key once it has — rather +/// than showing the same "no provider configured" state a never-configured workspace gets. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct FreeTierInfo { + /// The grant is spent: no provider is served and the user must bring their own key. + pub exhausted: bool, + /// Fraction of the grant consumed, 0.0..=1.0. A ratio, not a dollar amount — the + /// pricing model stays server-side. + pub used_ratio: f64, +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -423,10 +436,36 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Response-only: this same struct is the request body for saving a workspace's AI + /// config, and `skip_deserializing` is what stops a client from storing a forged + /// free-tier marker. Only the server sets it, per-request. + #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)] + pub free_tier: Option, /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] pub model_pricing: Option>, + /// Per-model context windows the chat budgets its history against, keyed + /// `provider:model` like `max_tokens_per_model`. Overrides the client's built-in + /// table, which a model served through a custom endpoint is usually missing from. + #[serde(skip_serializing_if = "Option::is_none")] + pub context_window_per_model: Option>, + /// Hides the Windmill AI assistant (chat, sessions, generation, completion, fixes) from + /// the workspace UI. Only the workspace's own row is consulted: the flag holds even when + /// the providers served come from the instance config or the free tier. AI agent steps + /// and the AI sandbox are unaffected, so the providers stay in force. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub copilot_disabled: bool, + /// Stops browsers from backing their AI sessions up to the workspace's object storage + /// (`ai_sessions.rs`). Read from the workspace's own row like `copilot_disabled`. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub sessions_storage_disabled: bool, + /// The server's sweep (`ai_sessions.rs`) deletes the backup of a session no push has + /// reached for this many days. The copies in members' browsers are untouched. Unset + /// keeps backups until the user deletes the session. Read from the workspace's own row + /// like `copilot_disabled`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sessions_retention_days: Option, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -463,6 +502,9 @@ impl ModelPriceOverride { } } +/// Ten years: past any plausible retention, and well within what a day count is turned into. +pub const MAX_SESSIONS_RETENTION_DAYS: u32 = 3650; + impl AIConfig { pub fn validate_model_pricing(&self) -> Result<()> { for (key, price) in self.model_pricing.iter().flatten() { @@ -471,6 +513,29 @@ impl AIConfig { Ok(()) } + pub fn validate_token_limits(&self) -> Result<()> { + for (entries, bounds) in [ + (&self.context_window_per_model, &CONTEXT_WINDOWS), + (&self.max_tokens_per_model, &OUTPUT_LIMITS), + ] { + for (key, tokens) in entries.iter().flatten() { + validate_token_limit(bounds, key, i64::from(*tokens)).map_err(Error::BadRequest)?; + } + } + Ok(()) + } + + pub fn validate_sessions_retention(&self) -> Result<()> { + match self.sessions_retention_days { + Some(days) if !(1..=MAX_SESSIONS_RETENTION_DAYS).contains(&days) => { + Err(Error::BadRequest(format!( + "AI session retention must be between 1 and {MAX_SESSIONS_RETENTION_DAYS} days (got {days})" + ))) + } + _ => Ok(()), + } + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() @@ -494,11 +559,18 @@ pub fn workspaced_service() -> Router { // could make the server allocate and parse an arbitrarily large one. // Sized well above a full batch of the shape below. .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ) + .nest( + "/shared_artifacts", + crate::ai_shared_artifacts::workspaced_service(), ); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); + #[cfg(feature = "parquet")] + let router = router.nest("/sessions", crate::ai_sessions::workspaced_service()); + router } @@ -1013,83 +1085,119 @@ async fn proxy( check_scopes(&authed, || format!("resources:read:{}", resource_path))?; } - let mut credentials = match workspace_cache { - Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => { - request_cache.credentials - } - _ => { - let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = - if let Some(resource_path) = forced_resource_path { - // forced resource path - (resource_path, false, w_id.clone(), None) - } else { - let workspace_ai_config = sqlx::query_scalar!( - "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // Set when serving the request through Windmill's free AI tier (the lent key). Holds + // the per-user concurrency lock and drives response metering. + let mut free_lease: Option = None; + let mut credentials = 'cred: { + match workspace_cache { + Some(request_cache) + if !request_cache.is_expired() && forced_resource_path.is_none() => + { + request_cache.credentials + } + _ => { + let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = + if let Some(resource_path) = forced_resource_path { + // forced resource path + (resource_path, false, w_id.clone(), None) + } else { + let workspace_ai_config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; - let (ai_config_value, resource_workspace, instance_ai_config_revision) = { - let ws_has_config = workspace_ai_config - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|config| config.has_providers()); + let (ai_config_value, resource_workspace, instance_ai_config_revision) = { + let ws_has_config = workspace_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .is_some_and(|config| config.has_providers()); - if ws_has_config { - (workspace_ai_config.unwrap(), w_id.clone(), None) - } else { - let instance_config = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = 'ai_config'" - ) - .fetch_optional(&db) - .await?; + if ws_has_config { + (workspace_ai_config.unwrap(), w_id.clone(), None) + } else { + let instance_config = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'ai_config'" + ) + .fetch_optional(&db) + .await?; - match instance_config { - Some(config) => ( - config, - "admins".to_string(), - Some(current_instance_ai_config_revision()), - ), - None => { - return Err(Error::internal_err( - "AI resource not configured".to_string(), - )); + let instance_has_config = + instance_config.as_ref().is_some_and(|v| { + serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|c| c.has_providers()) + }); + match instance_config { + // An instance `ai_config` row with no usable provider (e.g. `{}` + // or `{"providers":{}}`) is treated as unconfigured, exactly as + // build_copilot_settings_state does — otherwise its mere presence + // would suppress the free-tier fallback below. + Some(config) if instance_has_config => ( + config, + "admins".to_string(), + Some(current_instance_ai_config_revision()), + ), + _ => { + // Nothing configured: fall back to Windmill's free AI tier + // (EE-only) if a lent key is set and both the user's + // one-time grant and the instance's daily cap have room. + // Errors once the grant is spent, the day is capped, or the + // user already has a request in flight; None otherwise. + // Ineligible identities (e.g. service accounts) are refused + // inside the helper, so every path treats them alike. + let free = + crate::ai_free_tier_oss::resolve_free_tier_credentials( + &provider, + &db, + &ai_path, + &authed.email, + &body, + ) + .await?; + if let Some((free_credentials, lease)) = free { + free_lease = Some(lease); + break 'cred free_credentials; + } + return Err(Error::internal_err( + "AI resource not configured".to_string(), + )); + } } } + }; + + let mut ai_config = serde_json::from_value::(ai_config_value) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let provider_config = ai_config + .providers + .as_mut() + .and_then(|providers| providers.remove(&provider)) + .ok_or_else(|| { + Error::BadRequest(format!("Provider {:?} not configured", provider)) + })?; + + if provider_config.resource_path.is_empty() { + return Err(Error::BadRequest("Resource path is empty".to_string())); } + + ( + provider_config.resource_path, + true, + resource_workspace, + instance_ai_config_revision, + ) }; - let mut ai_config = serde_json::from_value::(ai_config_value) - .map_err(|e| Error::BadRequest(e.to_string()))?; - - let provider_config = ai_config - .providers - .as_mut() - .and_then(|providers| providers.remove(&provider)) - .ok_or_else(|| { - Error::BadRequest(format!("Provider {:?} not configured", provider)) - })?; - - if provider_config.resource_path.is_empty() { - return Err(Error::BadRequest("Resource path is empty".to_string())); - } - - ( - provider_config.resource_path, - true, - resource_workspace, - instance_ai_config_revision, - ) - }; - - // For user-specified resources, fetch through an RLS-scoped - // connection so PostgreSQL row-level security enforces the same - // folder/group boundaries as the regular resource API. For the - // workspace/instance ai_config path, the resource_path was already - // validated by an admin/devops user when configuring the workspace, - // so the raw pool is used. - let resource = if is_user_specified_resource { + // For user-specified resources, fetch through an RLS-scoped + // connection so PostgreSQL row-level security enforces the same + // folder/group boundaries as the regular resource API. For the + // workspace/instance ai_config path, the resource_path was already + // validated by an admin/devops user when configuring the workspace, + // so the raw pool is used. + let resource = if is_user_specified_resource { let mut tx = user_db.clone().begin(&authed).await?; let res = sqlx::query_scalar::<_, Option>>>( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", @@ -1112,38 +1220,45 @@ async fn proxy( .ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", resource_path)))? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", resource_path)))?; - let resource = serde_json::from_str::(resource.0.get()) - .map_err(|e| Error::BadRequest(e.to_string()))?; + let resource = serde_json::from_str::(resource.0.get()) + .map_err(|e| Error::BadRequest(e.to_string()))?; - // Enforce RLS on $var: resolution when the resource path was - // user-specified (X-Resource-Path header) so users can only read - // variables they have permission to access. - let enforce_authed = if is_user_specified_resource { - Some(&authed) - } else { - None - }; - let credentials = resolve_provider_credentials( - &provider, - &db, - &resource_workspace, - resource, - enforce_authed, - ) - .await?; - if save_to_cache { - AI_REQUEST_CACHE.insert( - (w_id.clone(), provider.clone()), - ExpiringProviderCredentials::new( - credentials.clone(), - instance_ai_config_revision, - ), - ); + // Enforce RLS on $var: resolution when the resource path was + // user-specified (X-Resource-Path header) so users can only read + // variables they have permission to access. + let enforce_authed = if is_user_specified_resource { + Some(&authed) + } else { + None + }; + let credentials = resolve_provider_credentials( + &provider, + &db, + &resource_workspace, + resource, + enforce_authed, + ) + .await?; + if save_to_cache { + AI_REQUEST_CACHE.insert( + (w_id.clone(), provider.clone()), + ExpiringProviderCredentials::new( + credentials.clone(), + instance_ai_config_revision, + ), + ); + } + credentials } - credentials } }; + // Free tier: pin the model and clamp max_tokens server-side before forwarding, + // since the request body is otherwise client-controlled. + if free_lease.is_some() { + body = crate::ai_free_tier_oss::enforce_free_tier_body(&body)?; + } + if let Some(fim_transform) = maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)? { @@ -1291,8 +1406,32 @@ async fn proxy( let status_code = response.status(); let headers = response.headers().clone(); + let is_sse = is_sse_response(&headers); + + // Free tier: reconcile the cost reserved up-front against what the response actually + // used, holding the per-user lock (via the lease) until it is recorded. The chat + // streams (SSE), where the usage report only arrives in the final chunk; the + // non-streaming JSON path is handled for completeness. + if let Some(lease) = free_lease { + let body = if is_sse { + axum::body::Body::from_stream(inject_keepalives( + Box::pin(crate::ai_free_tier_oss::meter_usage( + response.bytes_stream(), + db.clone(), + lease, + )), + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + let bytes = response.bytes().await.map_err(to_anyhow)?; + crate::ai_free_tier_oss::record_json_usage(db.clone(), lease, &bytes); + axum::body::Body::from(bytes) + }; + return Ok((status_code, headers, body)); + } + let stream = response.bytes_stream(); - let body = if is_sse_response(&headers) { + let body = if is_sse { axum::body::Body::from_stream(inject_keepalives( stream, Duration::from_secs(KEEPALIVE_INTERVAL_SECS), diff --git a/backend/windmill-api/src/ai_evals/datasets.rs b/backend/windmill-api/src/ai_evals/datasets.rs index da901276b0..8e1acb3d2a 100644 --- a/backend/windmill-api/src/ai_evals/datasets.rs +++ b/backend/windmill-api/src/ai_evals/datasets.rs @@ -190,6 +190,8 @@ pub async fn create_dataset( } tx.commit().await?; + windmill_common::feature_usage::log_feature_usage("ai_agent_eval", "dataset_created", ""); + Ok(format!("Created eval dataset {}", payload.path)) } diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs index e102abaf4c..5d282c7e5b 100644 --- a/backend/windmill-api/src/ai_evals/run.rs +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -707,6 +707,17 @@ pub async fn run_experiment( .await?; return Err(e); } + // Which state of the agent was measured is the whole key vocabulary: it is what separates + // running what is deployed from measuring edits or an older version. + windmill_common::feature_usage::log_feature_usage( + "ai_agent_eval", + "run", + match subject.kind { + EvalSubjectKind::Agent => "agent", + EvalSubjectKind::AgentDraft => "agent_draft", + EvalSubjectKind::AgentVersion => "agent_version", + }, + ); Ok(experiment_id.to_string()) } diff --git a/backend/windmill-api/src/ai_free_tier_oss.rs b/backend/windmill-api/src/ai_free_tier_oss.rs new file mode 100644 index 0000000000..7a183ff8ed --- /dev/null +++ b/backend/windmill-api/src/ai_free_tier_oss.rs @@ -0,0 +1,65 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ai_free_tier_ee::*; + +// Open-source build: Windmill's free AI tier does not exist. These stubs make the +// callers in `ai.rs` / `workspaces.rs` compile while disabling the feature entirely — +// `resolve_free_tier_credentials` never opts in, so the proxy falls through to its +// normal "AI resource not configured" path and the copilot stays hidden. +// +// Caller contract (enforced by the private impl, restated here for parity): the `email` +// passed to `resolve_free_tier_credentials` / `free_tier_copilot_config` MUST be the +// authenticated caller's own identity (an `ApiAuthed` email), never a client-supplied one — +// it selects whose lent-key grant is spent and whose usage is read. + +#[cfg(not(feature = "private"))] +use crate::ai::AIConfig; +#[cfg(not(feature = "private"))] +use crate::db::DB; +#[cfg(not(feature = "private"))] +use axum::body::Bytes; +#[cfg(not(feature = "private"))] +use windmill_ai::ai_providers::AIProvider; +#[cfg(not(feature = "private"))] +use windmill_ai::credentials::ProviderCredentials; +#[cfg(not(feature = "private"))] +use windmill_common::error::Result; + +#[cfg(not(feature = "private"))] +pub struct FreeTierLease; + +#[cfg(not(feature = "private"))] +pub async fn resolve_free_tier_credentials( + _provider: &AIProvider, + _db: &DB, + _ai_path: &str, + _email: &str, + _body: &Bytes, +) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn enforce_free_tier_body(body: &Bytes) -> Result { + Ok(body.clone()) +} + +#[cfg(not(feature = "private"))] +pub async fn free_tier_copilot_config(_db: &DB, _email: &str) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn record_json_usage(_db: DB, _lease: FreeTierLease, _bytes: &[u8]) {} + +#[cfg(not(feature = "private"))] +pub fn meter_usage( + upstream: S, + _db: DB, + _lease: FreeTierLease, +) -> impl futures::Stream> +where + S: futures::Stream> + Unpin, +{ + upstream +} diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs new file mode 100644 index 0000000000..65a8137c4e --- /dev/null +++ b/backend/windmill-api/src/ai_sessions.rs @@ -0,0 +1,1634 @@ +//! Lazily replicated backups of the browser's AI sessions in the workspace's object storage. +//! +//! The browser keeps the sessions in IndexedDB and pushes changed pieces here in batches; an +//! empty browser restores from what was pushed. The server owns the key layout, keeps the +//! caller's own prefix the only one it can reach, and encrypts every object with the +//! workspace key so bucket credentials do not read transcripts: +//! +//! ```text +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/head.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/chats/{cid}.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/artifacts.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/images/{sid}/{cid}/{iid} +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/index/{sid}/{epoch} +//! ``` +//! +//! The index marker is empty, written last by every push of the session, and is what a +//! listing reads: one object per session, whatever the session holds, its `last_modified` +//! the session's `updated_at`. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Path}, + routing::{get, post}, + Extension, Json, Router, +}; +use futures::{StreamExt, TryStreamExt}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::sync::Arc; +use windmill_api_auth::is_effectively_unscoped; +use windmill_api_workspaces::ai_session_backups::{ + fallback_store, generation_prefix, primary_store, storage_id, MAX_OBJECT_BYTES, +}; +use windmill_api_workspaces::workspaces::sessions_retention_days; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, +}; +use windmill_object_store::{build_object_store_client, object_store_error_to_error}; + +const PUSH_BODY_LIMIT: usize = MAX_OBJECT_BYTES; +/// A pull names at most MAX_PULL_IDS ids of 64 bytes; anything larger is not a pull. +const PULL_BODY_LIMIT: usize = 64 * 1024; +/// A pull answer larger than this hands the remaining ids back as `deferred`. +const PULL_RESPONSE_BUDGET: usize = 32 * 1024 * 1024; +const MAX_HEAD_BYTES: usize = 1024 * 1024; +/// What the cipher adds to a plaintext at most (a block of padding): an object stored at a +/// cap is that much larger than the cap when read back. +const CIPHER_PADDING: usize = 16; +/// The browser bounds an image to a 1568 px edge and re-encodes past 700 KB; this is +/// well above what that produces. +const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; +const MAX_PULL_IDS: usize = 20; +const MAX_PUSH_SESSIONS: usize = 100; +const MAX_REMOVED: usize = 200; +const MAX_CHATS_PER_ENTRY: usize = 100; +const MAX_IMAGES_PER_ENTRY: usize = 500; +const MAX_DELETES_PER_ENTRY: usize = 1000; +const MAX_OPERATIONS_PER_PUSH: usize = 4000; +/// Entries of listing metadata a pull holds per page of a session. +const MAX_LISTED_OBJECTS: usize = 5000; +/// Session markers a listing scans, and the newest sessions it answers with. +const MAX_LIST_SCAN: usize = 50_000; +const LIST_MAX: usize = 500; +const IO_CONCURRENCY: usize = 8; +/// Sessions the retention sweep deletes per workspace and pass at most; the rest wait for +/// the next pass. +const SWEEP_MAX_PER_WORKSPACE: usize = 1000; +/// Session-level advisory lock of the retention sweep: one server at a time runs it. +const SWEEP_LOCK_ID: i64 = 0x5745_4550_4149; +/// The name of the sweep's record next to a session's markers (see `Backend::sweep_key`). +const SWEEP_RECORD: &str = "sweep"; +/// The name of a split push's token next to a session's markers (see `Backend::push_key`). +const PUSH_TOKEN: &str = "push"; + +/// A marker modified before this is past a retention of `days`. +fn retention_cutoff(days: u32) -> chrono::DateTime { + chrono::Utc::now() - chrono::Duration::days(i64::from(days)) +} + +/// What a key under the `index/` prefix is. +enum IndexEntry { + /// The marker that lists the session, named by its epoch. + Marker(u32), + /// The retention sweep's record (see `Backend::sweep_key`). + Sweep, + /// The token of a push split over parts (see `Backend::push_key`). + Push, +} + +/// The session a key under the `index/` prefix belongs to, and what the key is. +fn index_entry<'a>(index: &ObjectPath, key: &'a ObjectPath) -> Option<(&'a str, IndexEntry)> { + // `Path` drops the trailing delimiter, so the remainder starts with one. + let rel = key.as_ref().strip_prefix(index.as_ref())?; + let (sid, name) = rel.trim_start_matches('/').split_once('/')?; + if sid.is_empty() { + return None; + } + let entry = match name { + SWEEP_RECORD => IndexEntry::Sweep, + PUSH_TOKEN => IndexEntry::Push, + epoch => IndexEntry::Marker(epoch.parse().ok()?), + }; + Some((sid, entry)) +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list)) + .route( + "/pull", + post(pull).layer(DefaultBodyLimit::max(PULL_BODY_LIMIT)), + ) + .route( + "/push", + post(push).layer(DefaultBodyLimit::max(PUSH_BODY_LIMIT)), + ) +} + +/// What reading an object yields. `Gone`: not there (deleted since the listing, or never +/// pushed). `Grown`: larger than expected, so replaced since the listing (or planted), and +/// left unread; a pull answers with a page ending before it rather than without it. +/// `Foreign`: it does not decrypt for this user (written under another user's or +/// workspace's key), and must not take the rest of the session down with it. +enum Read { + Text(String), + Gone, + Grown, + Foreign, +} + +/// The user's prefix in the workspace storage, plus what reads and writes it. +struct Backend { + store: Arc, + mc: MagicCrypt256, + prefix: String, + /// Name the storage and the generation the objects are under, for the browser's sync + /// state: a row recorded against another storage or generation is stale, a removal is + /// owed to the storage alone (a rotation deleted the older generation's copy anyway). + storage_id: String, + generation: i64, + /// `ai_config.sessions_retention_days`: a session whose marker is older is not listed, + /// whether or not the sweep has deleted it yet. + retention_days: Option, + /// The store is the instance object store standing in for a workspace without storage + /// of its own (`ai_session_backups::fallback_store`). + fallback: bool, +} + +impl Backend { + fn index_prefix(&self) -> ObjectPath { + ObjectPath::from(format!("{}/index/", self.prefix)) + } + + /// The moment a marker's modification time must reach to count as live, under the + /// workspace's retention; `None` without one. + fn retention_cutoff(&self) -> Option> { + self.retention_days.map(retention_cutoff) + } + + /// The marker that lists the session, named by the session's move count so that of a + /// session two workspaces list, the copy moved last is told from the listing alone. + fn index_key(&self, sid: &str, epoch: u32) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{epoch}", self.prefix)) + } + + fn index_session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/", self.prefix)) + } + + /// Written by the retention sweep before it deletes anything of a session, and deleted + /// last (`remove_session`): what finds a removal the sweep started and could not finish, + /// the markers being gone by then. Not an epoch, so nothing lists or pulls a session by it. + fn sweep_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{SWEEP_RECORD}", self.prefix)) + } + + /// The token of the push split over parts in progress, next to the markers so a removal + /// or the next whole push clears it with them, and the retention sweep, which walks the + /// markers, finds one a browser abandoned. + fn push_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{PUSH_TOKEN}", self.prefix)) + } + + fn session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/", self.prefix)) + } + + fn head_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/head.json", self.prefix)) + } + + fn chat_key(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/chats/{cid}.json", self.prefix)) + } + + fn artifacts_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/artifacts.json", self.prefix)) + } + + fn images_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/", self.prefix)) + } + + fn chat_images_prefix(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/", self.prefix)) + } + + fn image_key(&self, sid: &str, cid: &str, iid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/{iid}", self.prefix)) + } + + fn seal(&self, plaintext: &[u8]) -> Vec { + self.mc.encrypt_bytes_to_bytes(plaintext) + } + + /// Bytes written. + async fn put_sealed(&self, key: &ObjectPath, ciphertext: Vec) -> Result { + let written = ciphertext.len(); + self.store + .put(key, PutPayload::from(ciphertext)) + .await + .map_err(object_store_error_to_error)?; + Ok(written) + } + + async fn put(&self, key: &ObjectPath, plaintext: &[u8]) -> Result { + self.put_sealed(key, self.seal(plaintext)).await + } + + /// `max` is what the listing said the object holds, or the cap of its kind for one read + /// without a listing: checked before buffering, since whoever holds the bucket's + /// credentials can put anything at a predictable key. + async fn get(&self, key: &ObjectPath, max: usize) -> Result { + let result = match self.store.get(key).await { + Ok(result) => result, + Err(ObjectStoreError::NotFound { .. }) => return Ok(Read::Gone), + Err(e) => return Err(object_store_error_to_error(e)), + }; + let size = result.meta.size as usize; + // Larger than any push writes: planted, whatever the listing said, and skipped like + // an object of another key rather than retried like one that grew. + if size > MAX_OBJECT_BYTES { + tracing::warn!("AI session backup object {key} is larger than any push writes"); + return Ok(Read::Foreign); + } + if size > max { + return Ok(Read::Grown); + } + let bytes = result.bytes().await.map_err(object_store_error_to_error)?; + // The objects are JSON and data URLs: a wrong key's output failing UTF-8 tells it + // apart beyond the cipher's padding check, which a wrong key passes now and then. + match self + .mc + .decrypt_bytes_to_bytes(&bytes) + .ok() + .and_then(|plaintext| String::from_utf8(plaintext).ok()) + { + Some(text) => Ok(Read::Text(text)), + None => { + tracing::warn!("AI session backup object {key} does not decrypt for its reader"); + Ok(Read::Foreign) + } + } + } + + async fn delete(&self, key: &ObjectPath) -> Result<()> { + match self.store.delete(key).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// The entries under `prefix` past `after` in key order, as many as fit `budget` bytes; + /// `true` when more follow. Every key past `after` is seen and the MAX_LISTED_OBJECTS + /// smallest kept (a max-heap dropping its largest), since a page is defined by key + /// order and the store promises none; that cap is what bounds a pull's memory, a + /// session growing by valid pushes without limit. With `at_least_one`, the first entry + /// is taken whatever its size, so an answer owed the session makes progress on it (no + /// object exceeds the push body cap). + async fn list_within( + &self, + prefix: &ObjectPath, + after: Option<&ObjectPath>, + budget: usize, + at_least_one: bool, + ) -> Result<(Vec<(ObjectPath, usize)>, bool)> { + let mut kept: std::collections::BinaryHeap<(ObjectPath, usize)> = Default::default(); + let mut dropped = false; + let mut stream = match after { + Some(after) => self.store.list_with_offset(Some(prefix), after), + None => self.store.list(Some(prefix)), + }; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + kept.push((meta.location, meta.size as usize)); + if kept.len() > MAX_LISTED_OBJECTS { + kept.pop(); + dropped = true; + } + } + let mut entries = vec![]; + let mut total = 0; + for (key, size) in kept.into_sorted_vec() { + if total + size > budget && !(at_least_one && entries.is_empty()) { + return Ok((entries, true)); + } + total += size; + entries.push((key, size)); + } + Ok((entries, dropped)) + } + + /// Bytes written. Sealed up front so every stream item is owned: an item borrowing + /// from the request makes the future higher-ranked over that lifetime, which the + /// handler's `Send` bound cannot prove. + async fn put_all(&self, puts: Vec<(ObjectPath, Vec)>) -> Result { + futures::stream::iter(puts) + .map(|(key, ciphertext)| async move { self.put_sealed(&key, ciphertext).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_fold(0, |acc, n| async move { Ok::<_, Error>(acc + n) }) + .await + } + + async fn delete_all(&self, keys: Vec) -> Result<()> { + futures::stream::iter(keys) + .map(|key| async move { self.delete(&key).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_collect::>() + .await?; + Ok(()) + } + + /// A fingerprint of the session's marker and of everything listed under its two + /// prefixes (key, size, modification time, entity tag and version), combined as the + /// listing streams and in no particular order, so a session of any size costs bounded + /// memory. `None` for a session + /// the storage does not list. Taken before and after a page is read, so a page a push + /// changed under is read again; pages of one pull carry it, and the browser starts the + /// session over when it moved between two of them. + async fn listing_fingerprint(&self, sid: &str) -> Result> { + use std::hash::{DefaultHasher, Hash, Hasher}; + // The entity tag and version go in with the key, size and time: a store reports + // modification times coarsely, and an object rewritten at the same size within that + // grain would otherwise fingerprint the same. + fn fold( + acc: u64, + location: &str, + size: S, + modified: i64, + e_tag: Option<&str>, + version: Option<&str>, + ) -> u64 { + let mut hasher = DefaultHasher::new(); + (location, size, modified, e_tag, version).hash(&mut hasher); + acc.wrapping_add(hasher.finish()) + } + let mut acc = 0u64; + let mut listed = false; + for (marker, prefix) in [ + (true, self.index_session_prefix(sid)), + (false, self.session_prefix(sid)), + (false, self.images_prefix(sid)), + ] { + let mut stream = self.store.list(Some(&prefix)); + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + // The sweep's record is not a marker: a session it started removing is absent. + listed |= marker + && meta + .location + .filename() + .is_some_and(|name| name.parse::().is_ok()); + acc = fold( + acc, + meta.location.as_ref(), + meta.size, + meta.last_modified.timestamp_millis(), + meta.e_tag.as_deref(), + meta.version.as_deref(), + ); + } + } + if !listed { + return Ok(None); + } + Ok(Some(format!("{acc:016x}"))) + } + + async fn exists(&self, key: &ObjectPath) -> Result { + match self.store.head(key).await { + Ok(_) => Ok(true), + Err(ObjectStoreError::NotFound { .. }) => Ok(false), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// Deletes as the listing streams, so a prefix of any size costs bounded memory. + async fn delete_prefix(&self, prefix: &ObjectPath) -> Result<()> { + self.store + .list(Some(prefix)) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + self.delete(&meta.location).await + }) + .await + } +} + +/// Backups are the user's own browser state and nothing else may reach them: a job token +/// may carry an `on_behalf_of` identity, and every scoped token (guest, embed, app policy, +/// MCP) is minted for something narrower than the user's whole assistant history. +fn require_plain_user_token(authed: &ApiAuthed) -> Result<()> { + if authed.job_id.is_some() || !is_effectively_unscoped(authed.scopes.as_deref()) { + return Err(Error::PermissionDenied( + "AI session backups are only reachable with an unscoped user token".to_string(), + )); + } + Ok(()) +} + +/// Every key is assembled server-side from ids the browser mints (`createLongHash` and +/// `randomUUID` forms), so anything outside this alphabet is a forged id, not a real one. +fn require_valid_id(kind: &str, id: &str) -> Result<()> { + let ok = !id.is_empty() + && id.len() <= 64 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if !ok { + return Err(Error::BadRequest(format!("invalid {kind} id: {id:?}"))); + } + Ok(()) +} + +/// Images travel as base64 data URLs and are stored verbatim, so they serialize back into +/// a pull answer at exactly their stored size; anything else (control characters, +/// quotes) could grow several times under JSON escaping and defeat the pull budget. +fn require_data_url(data_url: &str) -> Result<()> { + let ok = data_url.len() <= MAX_IMAGE_BYTES + && data_url + .strip_prefix("data:") + .and_then(|rest| rest.split_once(";base64,")) + .is_some_and(|(mime, payload)| { + !mime.is_empty() + && mime.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'/' | b'.' | b'+' | b'-') + }) + && payload + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')) + }); + if !ok { + return Err(Error::BadRequest( + "an image must be a base64 data URL within the size cap".to_string(), + )); + } + Ok(()) +} + +fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<()> { + let text = raw.get(); + if !text.trim_start().starts_with('{') { + return Err(Error::BadRequest(format!("{kind} must be a JSON object"))); + } + if text.len() > max_bytes { + return Err(Error::BadRequest(format!( + "{kind} exceeds {max_bytes} bytes" + ))); + } + Ok(()) +} + +/// `None` when the workspace has nowhere to keep backups: no primary storage configured and +/// no instance store to stand in, or the admin switched them off. Both read as +/// `enabled: false` so the browser stops trying. +async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { + let (disabled, retention, generation, has_storage) = + sqlx::query_as::<_, (Option, Option, i64, bool)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, \ + ai_config->'sessions_retention_days', ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL \ + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_optional(db) + .await? + .unwrap_or((None, None, 0, false)); + if disabled.unwrap_or(false) { + return Ok(None); + } + let retention_days = sessions_retention_days(retention.as_ref()); + // Decided from the row the generation came from: the instance store is written only + // under a generation read while the workspace had no storage of its own, which + // configuring one moves past (`ai_session_backups`). + let (store, storage_id, fallback) = if has_storage { + let (_, resource) = + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; + let Some(resource) = resource else { + return Ok(None); + }; + ( + build_object_store_client(&resource).await?, + storage_id(&resource), + false, + ) + } else { + // The instance store stands in, under the same layout and the same key. + match fallback_store(db).await? { + Some(f) => (f.store, f.storage_id, true), + None => return Ok(None), + } + }; + let user = calculate_hash(&authed.email); + // Keyed per user, not per workspace: anyone who can write the bucket could otherwise copy + // another member's ciphertext under their own prefix and have `pull` decrypt it for them. + let key = get_workspace_key(w_id, db).await?; + let mc = crypt_from_key_with_suffix(&key, &user); + let prefix = format!("{}/{user}", generation_prefix(w_id, generation)); + Ok(Some(Backend { + store, + mc, + prefix, + storage_id, + generation, + retention_days, + fallback, + })) +} + +#[derive(Serialize)] +struct SessionListing { + id: String, + updated_at: chrono::DateTime, + /// The session's move count when this copy was pushed (see `PushedSession::epoch`). + epoch: u32, +} + +#[derive(Serialize)] +struct ListResponse { + enabled: bool, + /// The storage answered from, and the generation a key rotation bumps; a browser whose + /// sync state names another storage or generation starts over. + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + /// The storage is the instance store standing in for a workspace without one of its + /// own; a removal owed to it is retired by any answer from the workspace's own storage + /// once it has one (configuring it moved the generation past everything the workspace + /// left in any instance store). + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, + sessions: Vec, + /// The user has more sessions than the answer names. + #[serde(skip_serializing_if = "std::ops::Not::not")] + truncated: bool, +} + +/// A session is listed once a push entry of it landed whole (its marker is written last); +/// a push that failed before that left objects the listing does not name. One past the +/// workspace's retention is not listed either, whether or not the sweep has reached it, so a +/// browser never restores it. +async fn list( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_plain_user_token(&authed)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(ListResponse { + enabled: false, + storage_id: None, + backup_generation: None, + fallback: false, + sessions: vec![], + truncated: false, + })); + }; + let prefix = backend.index_prefix(); + let cutoff = backend.retention_cutoff(); + let mut stream = backend.store.list(Some(&prefix)); + // One marker per session, whatever the session holds: the newest LIST_MAX are kept as + // the scan goes (a min-heap drops the oldest), and the scan itself is bounded. + let mut newest: std::collections::BinaryHeap< + std::cmp::Reverse<(chrono::DateTime, u32, String)>, + > = Default::default(); + let mut scanned = 0; + let mut truncated = false; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + scanned += 1; + if scanned > MAX_LIST_SCAN { + truncated = true; + break; + } + if cutoff.is_some_and(|cutoff| meta.last_modified < cutoff) { + continue; + } + let Some((sid, IndexEntry::Marker(epoch))) = index_entry(&prefix, &meta.location) else { + continue; + }; + newest.push(std::cmp::Reverse(( + meta.last_modified, + epoch, + sid.to_string(), + ))); + if newest.len() > LIST_MAX { + newest.pop(); + truncated = true; + } + } + let mut sessions: Vec = newest + .into_iter() + .map(|std::cmp::Reverse((updated_at, epoch, id))| SessionListing { id, updated_at, epoch }) + .collect(); + sessions.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); + Ok(Json(ListResponse { + enabled: true, + storage_id: Some(backend.storage_id.clone()), + backup_generation: Some(backend.generation), + fallback: backend.fallback, + sessions, + truncated, + })) +} + +#[derive(Deserialize)] +struct PullRequest { + ids: Vec, + /// Picks the session an earlier answer cut up from where it stopped; `ids` then names + /// that session alone. + #[serde(default)] + resume: Option, +} + +/// Where a pull of a session that outgrew one answer picks up: the last key the earlier +/// answer carried, in the session's prefix or, once that one is done, in its images prefix. +#[derive(Serialize, Deserialize, Clone)] +struct PullCursor { + id: String, + images: bool, + after: String, +} + +#[derive(Serialize)] +struct PulledChat { + id: String, + record: Box, +} + +#[derive(Serialize, Deserialize)] +struct ImageObject { + chat_id: String, + id: String, + data_url: String, +} + +#[derive(Serialize)] +struct PulledSession { + id: String, + head: Box, + chats: Vec, + images: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + artifacts: Option>, + /// The session did not fit this answer whole: the rest follows a pull with this cursor. + #[serde(skip_serializing_if = "Option::is_none")] + next: Option, + /// A fingerprint of the session's listing (marker, and every key, size, modification + /// time, entity tag and version), so the browser tells that the backup changed between + /// the pages it assembled. + listing: String, + /// The backup kept changing while this page was read (a push landing object by object), + /// so the page may mix two versions: the browser starts the session over. + #[serde(skip_serializing_if = "std::ops::Not::not")] + moved: bool, +} + +/// How many times a page whose listing moved while it was read is read again before it is +/// handed over as `moved`. +const PULL_REREADS: usize = 3; + +#[derive(Serialize)] +struct PullResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, + sessions: Vec, + /// Ids that did not fit the response budget; ask for them again. + deferred: Vec, +} + +fn raw(kind: &str, text: String) -> Result> { + RawValue::from_string(text) + .map_err(|e| Error::internal_err(format!("stored {kind} is not JSON: {e}"))) +} + +enum PullStep { + Absent, + Deferred, + Fetched(PulledSession, usize), +} + +/// Fetch one session: its head, then every chat and artifact object under its prefix, and +/// as many of its images as the budget allows (a missing image hydrates to a placeholder +/// in the browser). Sizes come from the listings, so a session that would not fit is +/// deferred before anything of it is read, unless it is the first of the response, which +/// must carry something. `Absent` when it has no head. +async fn pull_session( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // A page read while a push lands object by object may mix two versions of the session: + // the listing is taken again once the page is read, and a page it moved under is read + // again, a few times, then handed over as such for the browser to start over. + for reread in 0..PULL_REREADS { + let step = pull_page(backend, sid, budget, first, resume).await?; + let PullStep::Fetched(mut page, size) = step else { + return Ok(step); + }; + if backend.listing_fingerprint(sid).await?.as_deref() == Some(page.listing.as_str()) { + return Ok(PullStep::Fetched(page, size)); + } + if reread + 1 == PULL_REREADS { + page.moved = true; + return Ok(PullStep::Fetched(page, size)); + } + } + unreachable!("a page is answered on the last reread") +} + +async fn pull_page( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // Taken before anything of the page is listed or read: an object landing after it is + // in the next page's fingerprint, whereas one landing after the reads but before a + // fingerprint taken then would have certified a page without it. A session the storage + // does not list (removed, or a whole push in progress) is absent. + let Some(listing) = backend.listing_fingerprint(sid).await? else { + return Ok(PullStep::Absent); + }; + let Read::Text(head) = backend + .get(&backend.head_key(sid), MAX_HEAD_BYTES + CIPHER_PADDING) + .await? + else { + return Ok(PullStep::Absent); + }; + let session_prefix = backend.session_prefix(sid); + let images_prefix = backend.images_prefix(sid); + let mut size = head.len(); + let mut chats = vec![]; + let mut artifacts = None; + let mut images = vec![]; + let mut next = None; + let cursor = |images: bool, after: String| PullCursor { id: sid.to_string(), images, after }; + // Sizes come from the listings, and the listings stop at the budget, so nothing is read + // past it even for the first session of the answer. One that outgrew it (chats + // accumulate over pushes) comes back in pages, in key order, each answer naming where + // the next picks up; the browser imports nothing before the last page. An object that + // grew since the listing (a push replaced it) ends the page just before it, and the + // answer names that spot: a new listing sizes it, whereas dropping it would import the + // session without it for good. + let in_images = resume.is_some_and(|c| c.images); + if !in_images { + let after = resume.map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &session_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + if cut && !first { + return Ok(PullStep::Deferred); + } + if cut { + next = entries + .last() + .map(|(key, _)| cursor(false, key.to_string())); + } + let to_read: Vec<(ObjectPath, usize, Option)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key + .as_ref() + .strip_prefix(session_prefix.as_ref()) + .unwrap_or_default() + .trim_start_matches('/'); + if rel == "artifacts.json" { + Some((key, bytes, None)) + } else { + let cid = rel + .strip_prefix("chats/")? + .strip_suffix(".json")? + .to_string(); + Some((key, bytes, Some(cid))) + } + }) + .collect(); + let reads: Vec<(ObjectPath, usize, Option, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + let mut before = resume.map(|c| c.after.clone()).unwrap_or_default(); + for (key, bytes, cid, read) in reads { + match (cid, read) { + (_, Read::Grown) => { + next = Some(cursor(false, before)); + break; + } + (None, Read::Text(text)) => { + artifacts = Some(raw("artifacts", text)?); + size += bytes; + } + (Some(cid), Read::Text(text)) => { + chats.push(PulledChat { id: cid, record: raw("chat", text)? }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + if next.is_none() { + let after = resume + .filter(|c| c.images) + .map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &images_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + let mut before = after.map(|a| a.to_string()).unwrap_or_default(); + if cut { + // An answer with no room for a single image names where it stood, so the pull + // owed the session alone picks it up there. + next = Some(cursor( + true, + entries + .last() + .map(|(key, _)| key.to_string()) + .unwrap_or_else(|| before.clone()), + )); + } + let to_read: Vec<(ObjectPath, usize, String, String)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key.as_ref().strip_prefix(images_prefix.as_ref())?; + let (cid, iid) = rel.trim_start_matches('/').split_once('/')?; + Some((key.clone(), bytes, cid.to_string(), iid.to_string())) + }) + .collect(); + let reads: Vec<(ObjectPath, usize, String, String, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid, iid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, iid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + for (key, bytes, chat_id, id, read) in reads { + match read { + Read::Grown => { + next = Some(cursor(true, before)); + break; + } + Read::Text(data_url) => { + images.push(ImageObject { chat_id, id, data_url }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + Ok(PullStep::Fetched( + PulledSession { + id: sid.to_string(), + head: raw("head", head)?, + chats, + images, + artifacts, + next, + listing, + moved: false, + }, + size, + )) +} + +async fn pull( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.ids.len() > MAX_PULL_IDS { + return Err(Error::BadRequest(format!( + "at most {MAX_PULL_IDS} sessions per pull" + ))); + } + for id in &req.ids { + require_valid_id("session", id)?; + } + if let Some(cursor) = &req.resume { + if req.ids.len() != 1 || req.ids[0] != cursor.id || cursor.after.len() > 1024 { + return Err(Error::BadRequest( + "a resumed pull names the resumed session alone".to_string(), + )); + } + } + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PullResponse { + enabled: false, + storage_id: None, + backup_generation: None, + fallback: false, + sessions: vec![], + deferred: vec![], + })); + }; + let mut sessions = vec![]; + let mut deferred = vec![]; + let mut budget = PULL_RESPONSE_BUDGET; + for sid in req.ids { + let resume = req.resume.as_ref().filter(|c| c.id == sid); + match pull_session(&backend, &sid, budget, sessions.is_empty(), resume).await? { + PullStep::Absent => {} + PullStep::Deferred => deferred.push(sid), + PullStep::Fetched(session, size) => { + budget = budget.saturating_sub(size); + sessions.push(session); + } + } + } + Ok(Json(PullResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + fallback: backend.fallback, + sessions, + deferred, + })) +} + +#[derive(Deserialize)] +struct PushedChat { + id: String, + record: Box, +} + +#[derive(Deserialize)] +struct ImageRef { + chat_id: String, + id: String, +} + +#[derive(Deserialize)] +struct PushedSession { + id: String, + #[serde(default)] + head: Option>, + #[serde(default)] + chats: Vec, + #[serde(default)] + images: Vec, + #[serde(default)] + artifacts: Option>, + #[serde(default)] + delete_chats: Vec, + #[serde(default)] + delete_images: Vec, + /// More parts of the session follow, in this push or a later one: the session is not + /// listed on this one. + #[serde(default)] + partial: bool, + /// A part of a push of the session whole: the head is on the part that opens it, which + /// replaces whatever the storage holds of the session, and every piece the browser has + /// is on one of them. An incremental part instead rides on a session the storage lists, + /// and is refused with `needs_whole` when it lists none. + #[serde(default)] + whole: bool, + /// A push split over several parts names itself on each of them with a token the + /// browser draws; the part that `opens` it unlists the session (a pull between two parts + /// would otherwise take a mix of old and new pieces for the backup) and the last part + /// lists it again. A later part is written only while that token is the one there, so a + /// part of a push another one superseded is refused with `needs_whole`. + #[serde(default)] + push: Option, + #[serde(default)] + opens: bool, + /// The session's move count (its record's `moves`), the marker that lists the session + /// is named by: a session moved to another workspace is listed by both until the old + /// copy's removal lands, and the copy with the higher count is the later one. An + /// incremental part rides on the marker of the same count. + #[serde(default)] + epoch: u32, +} + +#[derive(Deserialize)] +struct PushRequest { + /// The email the browser believes it is acting for. An in-place account switch can + /// leave a flush prepared for the previous user; the server refuses it rather than + /// filing that user's sessions under the caller's prefix. + owner: String, + #[serde(default)] + sessions: Vec, + #[serde(default)] + removed: Vec, +} + +#[derive(Serialize)] +struct PushResult { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + /// Nothing was written; the session must be pushed whole again: an incremental part + /// found no listed session to ride on (another device removed the backup, or a push + /// split over parts is in progress or was abandoned), or a later part of a push split + /// over parts found another push had superseded it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + needs_whole: bool, +} + +#[derive(Serialize)] +struct PushResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, + results: Vec, +} + +fn validate_push(req: &PushRequest) -> Result<()> { + if req.sessions.len() > MAX_PUSH_SESSIONS || req.removed.len() > MAX_REMOVED { + return Err(Error::BadRequest( + "too many sessions in one push".to_string(), + )); + } + // Every nested entry costs an object-store call (a deleted chat two), so the lists are + // bounded per entry and across the request; the browser sends far fewer. + let mut operations = req.removed.len(); + for s in &req.sessions { + if s.chats.len() > MAX_CHATS_PER_ENTRY + || s.images.len() > MAX_IMAGES_PER_ENTRY + || s.delete_chats.len() > MAX_DELETES_PER_ENTRY + || s.delete_images.len() > MAX_DELETES_PER_ENTRY + { + return Err(Error::BadRequest(format!( + "too many pieces for session {} in one push", + s.id + ))); + } + operations += s.chats.len() + s.images.len() + s.delete_chats.len() + s.delete_images.len(); + } + if operations > MAX_OPERATIONS_PER_PUSH { + return Err(Error::BadRequest("too many pieces in one push".to_string())); + } + for sid in &req.removed { + require_valid_id("session", sid)?; + } + for s in &req.sessions { + require_valid_id("session", &s.id)?; + if let Some(token) = &s.push { + require_valid_id("push", token)?; + } else if s.opens || s.partial { + // A part more parts follow belongs to a push split over parts, which names + // itself: without the token the session would stay listed between the parts. + return Err(Error::BadRequest(format!( + "session {} is pushed in parts with no push token", + s.id + ))); + } + if s.whole && (s.push.is_none() || s.opens) && s.head.is_none() { + return Err(Error::BadRequest(format!( + "session {} is pushed whole without its head", + s.id + ))); + } + if let Some(head) = &s.head { + require_json_object("head", head, MAX_HEAD_BYTES)?; + } + for c in &s.chats { + require_valid_id("chat", &c.id)?; + require_json_object("chat record", &c.record, PUSH_BODY_LIMIT)?; + } + if let Some(a) = &s.artifacts { + require_json_object("artifacts", a, PUSH_BODY_LIMIT)?; + } + for i in &s.images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + require_data_url(&i.data_url)?; + } + for c in &s.delete_chats { + require_valid_id("chat", c)?; + } + for i in &s.delete_images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + } + } + Ok(()) +} + +fn push_payload_bytes(req: &PushRequest) -> usize { + req.sessions + .iter() + .map(|s| { + s.head.as_ref().map_or(0, |h| h.get().len()) + + s.artifacts.as_ref().map_or(0, |a| a.get().len()) + + s.chats.iter().map(|c| c.record.get().len()).sum::() + + s.images.iter().map(|i| i.data_url.len()).sum::() + }) + .sum() +} + +/// Runs under the session's lock (see `lock_session`). The part that opens a whole push +/// (the one with the head) replaces the backup: the marker goes first, so nothing lists the +/// session until the last part, then everything else. An incremental part assumes the rest +/// of the session is in the storage, which a removal since would have taken, or a push +/// split over parts may still be bringing: it is refused unless the session is listed, and +/// unlists the session itself while it changes more than one object (a pull between two +/// writes would otherwise take a mix of old and new pieces for the backup). A push split +/// over parts names itself with a token: the part that opens it unlists the session and +/// writes the token, and a later part is written only while that token is +/// the one there, so two devices pushing the session at once cannot list a mix of their +/// pieces: the push that opened later wins, the other is refused and goes again. Every +/// refusal comes before anything of the part lands. Deletes run last, and the marker only +/// by the last part, so a push cut short never leaves a listed session pointing at chats +/// that are not there. +/// Bytes written, and whether the part was refused for the session to go whole. +async fn push_session(backend: &Backend, s: &PushedSession) -> Result<(usize, bool)> { + match &s.push { + Some(token) if !s.opens => { + match backend + .get(&backend.push_key(&s.id), token.len() + CIPHER_PADDING) + .await? + { + Read::Text(current) if current == *token => {} + _ => return Ok((0, true)), + } + } + _ => { + if s.whole { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + backend + .delete_prefix(&backend.session_prefix(&s.id)) + .await?; + backend.delete_prefix(&backend.images_prefix(&s.id)).await?; + } else { + if !backend.exists(&backend.index_key(&s.id, s.epoch)).await? { + return Ok((0, true)); + } + // Unlisted while more than one object changes (a push split over parts, or + // one part touching several pieces): a pull between two of the writes, or + // after one of them failed, would otherwise take a mix of old and new + // pieces for the backup. One object changing is one write. + let pieces = s.chats.len() + + s.images.len() + + usize::from(s.artifacts.is_some()) + + usize::from(s.head.is_some()) + + s.delete_chats.len() + + s.delete_images.len(); + if s.push.is_some() || pieces > 1 { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + } + } + if let Some(token) = &s.push { + backend + .put(&backend.push_key(&s.id), token.as_bytes()) + .await?; + } + } + } + let mut written = 0; + written += backend + .put_all( + s.images + .iter() + .map(|img| { + ( + backend.image_key(&s.id, &img.chat_id, &img.id), + backend.seal(img.data_url.as_bytes()), + ) + }) + .collect(), + ) + .await?; + written += backend + .put_all( + s.chats + .iter() + .map(|c| { + ( + backend.chat_key(&s.id, &c.id), + backend.seal(c.record.get().as_bytes()), + ) + }) + .collect(), + ) + .await?; + if let Some(a) = &s.artifacts { + written += backend + .put(&backend.artifacts_key(&s.id), a.get().as_bytes()) + .await?; + } + if let Some(h) = &s.head { + written += backend + .put(&backend.head_key(&s.id), h.get().as_bytes()) + .await?; + } + for cid in &s.delete_chats { + backend.delete(&backend.chat_key(&s.id, cid)).await?; + backend + .delete_prefix(&backend.chat_images_prefix(&s.id, cid)) + .await?; + } + backend + .delete_all( + s.delete_images + .iter() + .map(|i| backend.image_key(&s.id, &i.chat_id, &i.id)) + .collect(), + ) + .await?; + if s.partial { + return Ok((written, false)); + } + // Last, and by the last part only, so a session is listed once its whole entry landed. + backend + .store + .put(&backend.index_key(&s.id, s.epoch), PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + if s.push.is_some() { + backend.delete(&backend.push_key(&s.id)).await?; + } + Ok((written, false)) +} + +/// The markers go first so a removal cut short leaves nothing listed, then the head so +/// nothing pulls either, and no push takes it for a session still there (see `push_session`). +/// The retention sweep's record goes last (see `Backend::sweep_key`). +async fn remove_session(backend: &Backend, sid: &str) -> Result<()> { + let sweep = backend.sweep_key(sid); + backend + .store + .list(Some(&backend.index_session_prefix(sid))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| { + let sweep = &sweep; + async move { + if meta.location == *sweep { + return Ok(()); + } + backend.delete(&meta.location).await + } + }) + .await?; + backend.delete(&backend.head_key(sid)).await?; + backend.delete_prefix(&backend.session_prefix(sid)).await?; + backend.delete_prefix(&backend.images_prefix(sid)).await?; + backend.delete(&sweep).await +} + +/// One writer per session at a time, across servers: a push and a removal of the same +/// session interleaving object by object could leave a listed session missing pieces, or a +/// marker over nothing. The lock lives in a transaction that writes no rows; it is released +/// when the transaction ends. A wait past the timeout fails that entry only, and the +/// browser retries it with backoff. +async fn lock_session( + db: &DB, + backend: &Backend, + sid: &str, +) -> Result> { + let mut tx = db.begin().await?; + sqlx::query("SET LOCAL lock_timeout = '30s'") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0::int8))") + .bind(format!("ai_session_backup:{}/{sid}", backend.prefix)) + .execute(&mut *tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "another device is writing the backup of session {sid}; retried later ({e})" + )) + })?; + Ok(tx) +} + +async fn push_session_locked( + db: &DB, + backend: &Backend, + s: &PushedSession, +) -> Result<(usize, bool)> { + let tx = lock_session(db, backend, &s.id).await?; + let result = push_session(backend, s).await; + tx.commit().await?; + result +} + +async fn remove_session_locked(db: &DB, backend: &Backend, sid: &str) -> Result<()> { + let tx = lock_session(db, backend, sid).await?; + let result = remove_session(backend, sid).await; + tx.commit().await?; + result +} + +/// Deletes, in every workspace with `ai_config.sessions_retention_days`, the backups of the +/// sessions whose marker is older than that: the marker is rewritten by every push that +/// completes, so its modification time is the session's last activity as the storage clocks +/// it. For the monitor, on every server: a session-level advisory lock keeps one pass at a +/// time across them. The walk reads markers only, one object per session and nothing of what +/// the sessions hold, under each user's prefix in turn (`list_with_delimiter` names the +/// users), and deletes at most `SWEEP_MAX_PER_WORKSPACE` sessions per workspace and pass. A +/// session goes under its lock (`lock_session`), once its markers are listed again there and +/// still all older (see `sweep_session`). A removal cut short leaves the sweep's record next +/// to the markers, which the walk also collects, so the next pass finishes it. +pub async fn sweep_expired_ai_session_backups(db: &DB) { + let mut lock_conn = match db.acquire().await { + Ok(conn) => conn, + Err(e) => { + tracing::error!("AI session retention: could not acquire a connection: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(SWEEP_LOCK_ID) + .fetch_one(&mut *lock_conn) + .await + { + Ok(locked) => locked, + Err(e) => { + tracing::error!("AI session retention: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + if let Err(e) = sweep_workspaces(db).await { + tracing::error!("AI session retention sweep failed: {e:#}"); + } + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(SWEEP_LOCK_ID) + .execute(&mut *lock_conn) + .await + { + tracing::error!("AI session retention: advisory unlock failed: {e:#}"); + } +} + +async fn sweep_workspaces(db: &DB) -> Result<()> { + let workspaces = sqlx::query_as::<_, (String, Option, i64, bool)>( + "SELECT workspace_id, ai_config->'sessions_retention_days', ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL \ + FROM workspace_settings \ + WHERE ai_config->'sessions_retention_days' IS NOT NULL", + ) + .fetch_all(db) + .await?; + for (w_id, retention, generation, has_storage) in workspaces { + let Some(days) = sessions_retention_days(retention.as_ref()) else { + continue; + }; + match sweep_workspace(db, &w_id, days, generation, has_storage).await { + Ok(0) => {} + Ok(deleted) => tracing::info!( + "AI session retention deleted {deleted} session backups of {w_id} older than {days} days" + ), + Err(e) => tracing::warn!("AI session retention sweep of {w_id}: {e:#}"), + } + } + Ok(()) +} + +/// `has_storage` comes from the row `generation` was read from, as in `backend`: the instance +/// store is swept only under a generation read while the workspace had no storage of its own. +async fn sweep_workspace( + db: &DB, + w_id: &str, + days: u32, + generation: i64, + has_storage: bool, +) -> Result { + let resolved = if has_storage { + primary_store(db, w_id).await? + } else { + fallback_store(db).await? + }; + let Some(resolved) = resolved else { + return Ok(0); + }; + let key = get_workspace_key(w_id, db).await?; + let (store, storage_id) = (resolved.store, resolved.storage_id); + let cutoff = retention_cutoff(days); + let root = ObjectPath::from(generation_prefix(w_id, generation)); + let users = store + .list_with_delimiter(Some(&root)) + .await + .map_err(object_store_error_to_error)? + .common_prefixes; + let mut deleted = 0; + for user_prefix in users { + let Some(user) = user_prefix.filename() else { + continue; + }; + // The sweep decrypts nothing; the cipher is only what a `Backend` is made of. + let backend = Backend { + store: store.clone(), + mc: crypt_from_key_with_suffix(&key, user), + prefix: user_prefix.to_string(), + storage_id: storage_id.clone(), + generation, + retention_days: Some(days), + fallback: resolved.fallback, + }; + let index = backend.index_prefix(); + let mut markers = backend.store.list(Some(&index)); + let mut expired = std::collections::BTreeSet::new(); + while let Some(meta) = markers.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + let sid = match index_entry(&index, &meta.location) { + Some((sid, IndexEntry::Sweep)) => sid, + Some((sid, IndexEntry::Marker(_) | IndexEntry::Push)) + if meta.last_modified < cutoff => + { + sid + } + _ => continue, + }; + expired.insert(sid.to_string()); + if deleted + expired.len() >= SWEEP_MAX_PER_WORKSPACE { + break; + } + } + for sid in expired { + match sweep_session(db, &backend, &sid, cutoff).await { + Ok(true) => deleted += 1, + Ok(false) => {} + Err(e) => tracing::warn!( + "AI session retention left the backup of {sid} in {w_id} for the next pass: {e:#}" + ), + } + } + if deleted >= SWEEP_MAX_PER_WORKSPACE { + break; + } + } + Ok(deleted) +} + +/// True when the session was deleted. Under the session's lock its markers are listed again: +/// one a push renewed since the walk keeps the session. A session with none is left alone +/// while a push split over parts is between two of them (its token younger than the +/// retention) or it is gone, unless the sweep's record says a removal was started; an older +/// token is a split push a browser abandoned, whose landed parts nothing lists. The record is +/// written before anything is deleted and removed last, so a removal cut short is found again +/// by the next pass. +async fn sweep_session( + db: &DB, + backend: &Backend, + sid: &str, + cutoff: chrono::DateTime, +) -> Result { + let tx = lock_session(db, backend, sid).await?; + let result = async { + let (sweep, push) = (backend.sweep_key(sid), backend.push_key(sid)); + let mut entries = backend.store.list(Some(&backend.index_session_prefix(sid))); + let (mut listed, mut renewed, mut started, mut abandoned) = (false, false, false, false); + while let Some(meta) = entries.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + if meta.location == sweep { + started = true; + } else if meta.location == push { + abandoned = meta.last_modified < cutoff; + } else { + listed = true; + renewed |= meta.last_modified >= cutoff; + } + } + if renewed { + // A push listed the session again over a removal cut short before its markers + // went, which had deleted nothing else. + if started { + backend.delete(&sweep).await?; + } + return Ok(false); + } + if !listed && !started && !abandoned { + return Ok(false); + } + if !started { + backend + .store + .put(&sweep, PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + } + remove_session(backend, sid).await?; + Ok(true) + } + .await; + tx.commit().await?; + result +} + +async fn push( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.owner != authed.email { + return Err(Error::Generic( + http::StatusCode::CONFLICT, + "this push was prepared for another user".to_string(), + )); + } + validate_push(&req)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PushResponse { + enabled: false, + storage_id: None, + backup_generation: None, + fallback: false, + results: vec![], + })); + }; + #[cfg(not(feature = "enterprise"))] + { + let remaining = + crate::job_helpers_oss::ce_storage_quota_remaining(&db, &w_id, None).await?; + if push_payload_bytes(&req) as i64 > remaining { + return Err(Error::QuotaExceeded( + "the workspace storage quota leaves no room for this AI session backup".to_string(), + )); + } + } + #[cfg(feature = "enterprise")] + let _ = push_payload_bytes(&req); + + let mut results = Vec::with_capacity(req.sessions.len() + req.removed.len()); + let mut written: usize = 0; + // A session split into several entries is listed by the last: once one part failed, the + // later ones are not written, or the marker would list a session missing a part. + let mut failed: std::collections::HashSet<&str> = Default::default(); + for s in &req.sessions { + let (error, needs_whole) = if failed.contains(s.id.as_str()) { + ( + Some("an earlier part of this session in the push failed".to_string()), + false, + ) + } else { + match push_session_locked(&db, &backend, s).await { + Ok((n, needs_whole)) => { + written += n; + (None, needs_whole) + } + Err(e) => { + tracing::warn!("AI session backup push failed for {} in {w_id}: {e}", s.id); + failed.insert(&s.id); + (Some(e.to_string()), false) + } + } + }; + results.push(PushResult { id: s.id.clone(), error, needs_whole }); + } + for sid in &req.removed { + let error = remove_session_locked(&db, &backend, sid) + .await + .err() + .map(|e| { + tracing::warn!("AI session backup removal failed for {sid} in {w_id}: {e}"); + e.to_string() + }); + results.push(PushResult { id: sid.clone(), error, needs_whole: false }); + } + // Overwrites and deletes make this an over-count; the periodic recount the quota check + // schedules once usage is stale settles it. Bytes in the instance store count under a + // name of their own, which the recount lists there. + #[cfg(not(feature = "enterprise"))] + if written > 0 { + let storage = if backend.fallback { + windmill_api_workspaces::ai_session_backups::FALLBACK_STORAGE + } else { + windmill_object_store::DEFAULT_STORAGE + }; + crate::job_helpers_oss::bump_storage_usage(&db, &w_id, storage, written as i64).await; + } + #[cfg(feature = "enterprise")] + let _ = written; + Ok(Json(PushResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + fallback: backend.fallback, + results, + })) +} diff --git a/backend/windmill-api/src/ai_shared_artifacts.rs b/backend/windmill-api/src/ai_shared_artifacts.rs new file mode 100644 index 0000000000..36ad0ecd2e --- /dev/null +++ b/backend/windmill-api/src/ai_shared_artifacts.rs @@ -0,0 +1,322 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Read-only copies of AI session artifacts, shared with the workspace by their author. +//! +//! Artifacts live in the author's browser; a row here exists only because the author asked +//! for a link. Every read filters on the retention window as well as the monitor sweeping +//! it, so a share past its window is never served in the gap before the sweep reaches it. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Extension, Json, Path, Query}, + routing::{delete, get, post}, + Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + ai_shared_artifact_retention_secs, + error::{Error, JsonResult, Result}, +}; + +/// The frontend's `MAX_ARTIFACT_BYTES`: no artifact the viewer holds is larger. +const MAX_CONTENT_BYTES: usize = 256 * 1024; +const MAX_NAME_CHARS: usize = 255; +const MAX_ARTIFACT_ID_CHARS: usize = 255; +/// JSON escapes a control character into six bytes, so a full-size artifact made entirely of +/// them still fits. +const SHARE_BODY_LIMIT: usize = MAX_CONTENT_BYTES * 6 + 64 * 1024; + +pub fn workspaced_service() -> Router { + Router::new() + .route( + "/share", + post(share_artifact).layer(DefaultBodyLimit::max(SHARE_BODY_LIMIT)), + ) + .route("/status", get(get_share_status)) + .route("/get/{id}", get(get_shared_artifact)) + .route("/delete/{id}", delete(unshare_artifact)) +} + +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "lowercase")] +enum ArtifactKind { + Md, + Html, +} + +impl ArtifactKind { + fn as_str(self) -> &'static str { + match self { + ArtifactKind::Md => "md", + ArtifactKind::Html => "html", + } + } +} + +#[derive(Deserialize)] +struct ShareArtifact { + artifact_id: String, + name: String, + kind: ArtifactKind, + version: i32, + content: String, +} + +#[derive(Serialize)] +struct SharedArtifactInfo { + id: Uuid, + name: String, + kind: String, + version: i32, + created_by: String, + shared_at: DateTime, + expires_at: DateTime, +} + +#[derive(Serialize)] +struct SharedArtifact { + #[serde(flatten)] + info: SharedArtifactInfo, + content: String, + /// Whether the caller may stop sharing it: its author, or a workspace admin. + can_unshare: bool, +} + +#[derive(Serialize)] +struct ShareStatus { + retention_secs: i64, + #[serde(skip_serializing_if = "Option::is_none")] + share: Option, +} + +#[derive(Deserialize)] +struct ShareStatusQuery { + artifact_id: String, +} + +fn expires_at(shared_at: DateTime, retention_secs: i64) -> DateTime { + shared_at + chrono::Duration::seconds(retention_secs) +} + +/// The browser-side artifact id, as every handler that takes one must check it: it is compared +/// against a `VARCHAR(255)` column, and Postgres answers a NUL in a text parameter with an +/// opaque 500. +fn check_artifact_id(artifact_id: &str) -> Result<()> { + if artifact_id.is_empty() || artifact_id.chars().count() > MAX_ARTIFACT_ID_CHARS { + return Err(Error::BadRequest(format!( + "Artifact id must be between 1 and {MAX_ARTIFACT_ID_CHARS} characters" + ))); + } + if artifact_id.contains('\0') { + return Err(Error::BadRequest( + "Artifact id cannot contain NUL characters".to_string(), + )); + } + Ok(()) +} + +/// Share an artifact, or move the caller's existing link for it to this content. Re-sharing +/// keeps the link and restarts its retention window. +async fn share_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> JsonResult { + let name = payload.name.trim(); + if name.is_empty() || name.chars().count() > MAX_NAME_CHARS { + return Err(Error::BadRequest(format!( + "Artifact name must be between 1 and {MAX_NAME_CHARS} characters" + ))); + } + check_artifact_id(&payload.artifact_id)?; + if payload.content.len() > MAX_CONTENT_BYTES { + return Err(Error::BadRequest(format!( + "Artifact content is {} bytes, above the {MAX_CONTENT_BYTES} byte limit", + payload.content.len() + ))); + } + if payload.version < 1 { + return Err(Error::BadRequest( + "Artifact version must be at least 1".to_string(), + )); + } + // Postgres rejects NUL in text columns with an opaque 500. + if name.contains('\0') || payload.content.contains('\0') { + return Err(Error::BadRequest( + "Artifact name and content cannot contain NUL characters".to_string(), + )); + } + + let mut tx = db.begin().await?; + let row = sqlx::query!( + r#"INSERT INTO ai_shared_artifact + (workspace_id, artifact_id, email, created_by, name, kind, version, content) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE + SET created_by = EXCLUDED.created_by, + name = EXCLUDED.name, + kind = EXCLUDED.kind, + version = EXCLUDED.version, + content = EXCLUDED.content, + shared_at = now() + RETURNING id, shared_at, (xmax = 0) AS "inserted!""#, + &w_id, + &payload.artifact_id, + &authed.email, + &authed.username, + name, + payload.kind.as_str(), + payload.version, + &payload.content, + ) + .fetch_one(&mut *tx) + .await?; + + let id = row.id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.share", + if row.inserted { + ActionKind::Create + } else { + ActionKind::Update + }, + &w_id, + Some(&id), + Some([("name", name)].into()), + ) + .await?; + tx.commit().await?; + + Ok(Json(SharedArtifactInfo { + id: row.id, + name: name.to_string(), + kind: payload.kind.as_str().to_string(), + version: payload.version, + created_by: authed.username.clone(), + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, ai_shared_artifact_retention_secs()), + })) +} + +/// The caller's own live share of one of their artifacts, if any, and how long a share lasts. +async fn get_share_status( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + check_artifact_id(&query.artifact_id)?; + let retention_secs = ai_shared_artifact_retention_secs(); + let share = sqlx::query!( + "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact + WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3 + AND shared_at > now() - ($4::bigint::text || ' s')::interval", + &w_id, + &authed.email, + &query.artifact_id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .map(|r| SharedArtifactInfo { + id: r.id, + name: r.name, + kind: r.kind, + version: r.version, + created_by: r.created_by, + shared_at: r.shared_at, + expires_at: expires_at(r.shared_at, retention_secs), + }); + + Ok(Json(ShareStatus { retention_secs, share })) +} + +/// Any member of the workspace may read a live share: that is what sharing it granted. +async fn get_shared_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> JsonResult { + let retention_secs = ai_shared_artifact_retention_secs(); + let row = sqlx::query!( + "SELECT id, email, name, kind, version, created_by, content, shared_at + FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 + AND shared_at > now() - ($3::bigint::text || ' s')::interval", + &w_id, + id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found: it may have expired or been unshared" + )) + })?; + + Ok(Json(SharedArtifact { + can_unshare: row.email == authed.email || authed.is_admin, + content: row.content, + info: SharedArtifactInfo { + id: row.id, + name: row.name, + kind: row.kind, + version: row.version, + created_by: row.created_by, + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, retention_secs), + }, + })) +} + +async fn unshare_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> Result { + let mut tx = db.begin().await?; + let name = sqlx::query_scalar!( + "DELETE FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool) + RETURNING name", + &w_id, + id, + &authed.email, + authed.is_admin, + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found, or not shared by you" + )) + })?; + + let id_str = id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.unshare", + ActionKind::Delete, + &w_id, + Some(&id_str), + Some([("name", name.as_str())].into()), + ) + .await?; + tx.commit().await?; + + Ok(format!("Stopped sharing {name}")) +} diff --git a/backend/windmill-api/src/ai_skills.rs b/backend/windmill-api/src/ai_skills.rs deleted file mode 100644 index 11d99d55da..0000000000 --- a/backend/windmill-api/src/ai_skills.rs +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2026 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::db::{ApiAuthed, DB}; -use axum::{ - extract::{Extension, Json, Path}, - routing::{delete, get, post}, - Router, -}; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::require_admin, -}; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_skills)) - .route("/get/{name}", get(get_skill)) - .route("/upload", post(upload_skills)) - .route("/delete/{name}", delete(delete_skill)) -} - -/// Cheap listing surfaced in the AI chat system prompt — no `instructions` body. -#[derive(Serialize)] -pub struct SkillListItem { - pub name: String, - pub description: String, -} - -/// Full skill, including the SKILL.md body, fetched on demand by `read_skill`. -#[derive(Serialize)] -pub struct Skill { - pub name: String, - pub description: String, - pub instructions: String, -} - -#[derive(Deserialize)] -pub struct UploadSkills { - pub skills: Vec, -} - -#[derive(Deserialize)] -pub struct SkillUpload { - pub name: String, - pub description: String, - pub instructions: String, -} - -const MAX_SKILLS_PER_UPLOAD: usize = 50; -// Every stored skill's name + description is advertised in the global AI chat -// system prompt, so bound the total a workspace can accumulate across uploads. -const MAX_SKILLS_PER_WORKSPACE: usize = 100; -// `name` and `description` follow the Claude SKILL.md spec -// (https://platform.claude.com/docs/en/agents-and-tools/agent-skills): both are -// loaded into the AI chat system prompt and `name` is the model-facing skill id, -// so matching the upstream limits keeps skills portable with Claude Code. -const MAX_SKILL_NAME_CHARS: usize = 64; -const MAX_SKILL_DESCRIPTION_CHARS: usize = 1_024; -// Not a spec field — a payload bound on the SKILL.md body, so measured in bytes. -const MAX_SKILL_INSTRUCTIONS_BYTES: usize = 64 * 1024; - -fn validate_skill(skill: &SkillUpload) -> Result<()> { - let name = skill.name.trim(); - if name.is_empty() || name.chars().count() > MAX_SKILL_NAME_CHARS { - return Err(Error::BadRequest(format!( - "skill name must be between 1 and {MAX_SKILL_NAME_CHARS} characters, got {:?}", - skill.name - ))); - } - if !name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - return Err(Error::BadRequest(format!( - "skill name {name:?} must only contain lowercase letters, digits or '-'" - ))); - } - if skill.description.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} is missing a description (the SKILL.md frontmatter `description`)" - ))); - } - if skill.description.chars().count() > MAX_SKILL_DESCRIPTION_CHARS { - return Err(Error::BadRequest(format!( - "skill {name:?} description must be at most {MAX_SKILL_DESCRIPTION_CHARS} characters" - ))); - } - if skill.instructions.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} has an empty SKILL.md body" - ))); - } - if skill.instructions.len() > MAX_SKILL_INSTRUCTIONS_BYTES { - return Err(Error::BadRequest(format!( - "skill {name:?} instructions must be at most {MAX_SKILL_INSTRUCTIONS_BYTES} bytes" - ))); - } - Ok(()) -} - -/// Collect the trimmed skill names, rejecting duplicates within a single upload. -/// The insert upserts by name, so a duplicate would silently keep only the last -/// and make the reported/audited count wrong. -fn collect_upload_names(skills: &[SkillUpload]) -> Result> { - let mut names = Vec::with_capacity(skills.len()); - let mut seen = HashSet::with_capacity(skills.len()); - for skill in skills { - let name = skill.name.trim().to_string(); - if !seen.insert(name.clone()) { - return Err(Error::BadRequest(format!( - "duplicate skill name {name:?} in upload" - ))); - } - names.push(name); - } - Ok(names) -} - -/// Reject an upload that would push the workspace past `MAX_SKILLS_PER_WORKSPACE`. -/// Uploads upsert, so names already present (`replacing`) don't count as new. -fn check_workspace_skill_capacity( - existing_total: i64, - replacing: i64, - upload_count: usize, -) -> Result<()> { - let new_count = upload_count as i64 - replacing; - if existing_total + new_count > MAX_SKILLS_PER_WORKSPACE as i64 { - return Err(Error::BadRequest(format!( - "workspace cannot store more than {MAX_SKILLS_PER_WORKSPACE} skills" - ))); - } - Ok(()) -} - -async fn list_skills( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query!( - "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - &w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json( - rows.into_iter() - .map(|r| SkillListItem { name: r.name, description: r.description }) - .collect(), - )) -} - -async fn get_skill( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let row = sqlx::query!( - "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - row.map(|r| { - Json(Skill { name: r.name, description: r.description, instructions: r.instructions }) - }) - .ok_or_else(|| Error::NotFound(format!("no skill named {name:?} in workspace {w_id}"))) -} - -/// Bulk upsert the uploaded skills by name. Existing skills not in the payload -/// are left untouched — removal goes through `delete_skill`. -async fn upload_skills( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(payload): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - if payload.skills.is_empty() { - return Err(Error::BadRequest("no skills to upload".to_string())); - } - if payload.skills.len() > MAX_SKILLS_PER_UPLOAD { - return Err(Error::BadRequest(format!( - "cannot upload more than {MAX_SKILLS_PER_UPLOAD} skills at a time" - ))); - } - for skill in &payload.skills { - validate_skill(skill)?; - } - let names = collect_upload_names(&payload.skills)?; - - let mut tx = db.begin().await?; - let counts = sqlx::query!( - r#"SELECT - COUNT(*)::bigint AS "total!", - COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS "replacing!" - FROM ai_skill - WHERE workspace_id = $1"#, - &w_id, - &names - ) - .fetch_one(&mut *tx) - .await?; - check_workspace_skill_capacity(counts.total, counts.replacing, names.len())?; - - for (skill, name) in payload.skills.iter().zip(names.iter()) { - sqlx::query!( - r#"INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by) - VALUES ($1, $2, $3, $4, now(), $5) - ON CONFLICT (workspace_id, name) DO UPDATE - SET description = EXCLUDED.description, - instructions = EXCLUDED.instructions, - edited_at = now(), - edited_by = EXCLUDED.edited_by"#, - &w_id, - name, - skill.description, - skill.instructions, - &authed.username, - ) - .execute(&mut *tx) - .await?; - } - - let audit_resource = names.join(","); - audit_log( - &mut *tx, - &authed, - "ai_skills.upload", - ActionKind::Update, - &w_id, - Some(&audit_resource), - Some([("skill_count", &names.len().to_string()[..])].into()), - ) - .await?; - tx.commit().await?; - - Ok(format!( - "Uploaded {} skill(s) to workspace {}", - payload.skills.len(), - &w_id - )) -} - -async fn delete_skill( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - let deleted = sqlx::query_scalar!( - "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - - if deleted.is_none() { - tx.commit().await?; - return Err(Error::NotFound(format!( - "no skill named {name:?} in workspace {w_id}" - ))); - } - - audit_log( - &mut *tx, - &authed, - "ai_skills.delete", - ActionKind::Delete, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Deleted skill {name} from workspace {w_id}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn skill() -> SkillUpload { - SkillUpload { - name: "test-skill".to_string(), - description: "Useful for tests".to_string(), - instructions: "# Test\n\nDo the thing.".to_string(), - } - } - - #[test] - fn validate_skill_rejects_oversized_description() { - let mut skill = skill(); - skill.description = "x".repeat(MAX_SKILL_DESCRIPTION_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_instructions() { - let mut skill = skill(); - skill.instructions = "x".repeat(MAX_SKILL_INSTRUCTIONS_BYTES + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_name() { - let mut skill = skill(); - skill.name = "a".repeat(MAX_SKILL_NAME_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_non_slug_name() { - // Uppercase, underscore, space and punctuation are all outside the - // Claude SKILL.md `[a-z0-9-]` name charset. - for bad in ["My-Skill", "my_skill", "my skill", "skill!"] { - let mut skill = skill(); - skill.name = bad.to_string(); - - assert!( - matches!(validate_skill(&skill), Err(Error::BadRequest(_))), - "{bad:?} should be rejected" - ); - } - } - - #[test] - fn validate_skill_counts_description_in_characters() { - // 1024 two-byte chars exceed the byte limit but sit exactly on the - // character limit, so they must be accepted. - let mut skill = skill(); - skill.description = "é".repeat(MAX_SKILL_DESCRIPTION_CHARS); - - assert!(validate_skill(&skill).is_ok()); - } - - #[test] - fn workspace_capacity_allows_replacement_at_cap() { - // Already at the cap, but the upload only replaces an existing skill. - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(check_workspace_skill_capacity(at_cap, 1, 1).is_ok()); - } - - #[test] - fn workspace_capacity_rejects_new_skill_over_cap() { - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(matches!( - check_workspace_skill_capacity(at_cap, 0, 1), - Err(Error::BadRequest(_)) - )); - } - - #[test] - fn collect_upload_names_trims_and_collects() { - let names = collect_upload_names(&[skill()]).unwrap(); - assert_eq!(names, vec!["test-skill".to_string()]); - } - - #[test] - fn collect_upload_names_rejects_duplicates() { - // Names are compared after trimming, so whitespace can't smuggle a dup in. - let dup = SkillUpload { name: " test-skill ".to_string(), ..skill() }; - assert!(matches!( - collect_upload_names(&[skill(), dup]), - Err(Error::BadRequest(_)) - )); - } -} diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e5b5f019eb..717be4ad0f 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -17,6 +17,7 @@ use std::str::FromStr; use uuid::Uuid; use windmill_common::cache; use windmill_common::error::Error; +use windmill_common::flows::{ApprovalSkin, Suspend}; use windmill_common::jobs::JobKind; use windmill_common::scripts::ScriptHash; @@ -94,6 +95,17 @@ pub struct ApprovalFormDetails { pub message_str: String, pub urls: ResumeUrls, pub schema: Option, + pub skin: ApprovalSkin, +} + +/// The suspended step an approval message is about, and the flow run it belongs to. +struct ApprovalStep { + created_by: String, + created_at: chrono::NaiveDateTime, + script_path: Option, + parent_job_id: Option, + args: Option>>, + suspend: Option, } #[allow(dead_code)] @@ -205,6 +217,90 @@ pub async fn get_approval_form_details( tracing::debug!("Job ID: {:?}", job_id); + let ApprovalStep { created_by, created_at, script_path, parent_job_id, args, suspend } = + fetch_approval_step(&db, w_id, job_id, flow_step_id).await?; + + let schema = suspend.as_ref().map(|suspend| ResumeFormRow { + resume_form: suspend.resume_form.clone(), + hide_cancel: suspend.hide_cancel, + }); + let skin = suspend.and_then(|s| s.skin).unwrap_or_default(); + + let bold_format = match format { + MessageFormat::Slack => "*{}*", + MessageFormat::Teams => "**{}**", + }; + + let message_str = match skin { + ApprovalSkin::Detailed => { + let args_str = args.map_or("None".to_string(), |a| { + serde_json::from_str::(a.get()) + .ok() + .and_then(|v| serde_json::to_string_pretty(&v).ok()) + .unwrap_or_else(|| a.get().to_string()) + }); + let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); + let script_path_str = script_path.as_deref().unwrap_or("None"); + + let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); + + let mut message_str = format!( + "A workflow has been suspended and is waiting for approval:\n\n\ + {}: {created_by}\n\n\ + {}: {created_at_formatted}\n\n\ + {}: {script_path_str}\n\n\ + {}:\n```\n{args_str}\n```\n\n\ + {}: {parent_job_id_str}\n\n", + bold_format.replace("{}", "Created by"), + bold_format.replace("{}", "Created at"), + bold_format.replace("{}", "Script path"), + bold_format.replace("{}", "Args"), + bold_format.replace("{}", "Flow ID") + ); + + // Append custom message if provided + if let Some(msg) = message { + message_str.push_str(msg); + } + message_str + } + ApprovalSkin::Minimal => format!( + "{}\n\n{}: {created_by}", + message.unwrap_or("Your approval is requested."), + bold_format.replace("{}", "Requested by"), + ), + }; + + tracing::debug!("Schema: {:#?}", schema); + + Ok(ApprovalFormDetails { message_str, urls, schema, skin }) +} + +/// The skin of the approval step `flow_step_id` of the flow running `job_id`. Falls back to +/// the detailed skin when the step cannot be resolved, so a message is still sent. +/// Reads through the unrestricted pool without an authorization check of its own: only the +/// skin, which is not sensitive, leaves this function. +pub(crate) async fn get_approval_step_skin( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: &str, +) -> ApprovalSkin { + match fetch_approval_step(db, w_id, job_id, Some(flow_step_id)).await { + Ok(step) => step.suspend.and_then(|s| s.skin).unwrap_or_default(), + Err(e) => { + tracing::warn!("Could not resolve approval step {flow_step_id} of job {job_id}: {e}"); + ApprovalSkin::default() + } + } +} + +async fn fetch_approval_step( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: Option<&str>, +) -> Result { // TODO: do we have a helper function for this? let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( "WITH job_info AS ( @@ -240,17 +336,17 @@ pub async fn get_approval_form_details( job_id, &w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await .map_err(|e| Error::BadRequest(e.to_string()))? .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; - let flow_data = match cache::job::fetch_flow(&db, &job_kind, script_hash).await { + let flow_data = match cache::job::fetch_flow(db, &job_kind, script_hash).await { Ok(data) => data, Err(_) => { if let Some(parent_job_id) = parent_job_id.as_ref() { - cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? + cache::job::fetch_preview_flow(db, parent_job_id, raw_flow).await? } else { return Err(Error::BadRequest( "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), @@ -265,49 +361,12 @@ pub async fn get_approval_form_details( tracing::debug!("Module: {:#?}", module); - let schema = module.and_then(|module| { - module.suspend.as_ref().map(|suspend| ResumeFormRow { - resume_form: suspend.resume_form.clone(), - hide_cancel: suspend.hide_cancel, - }) - }); - - let args_str = args.map_or("None".to_string(), |a| { - serde_json::from_str::(a.get()) - .ok() - .and_then(|v| serde_json::to_string_pretty(&v).ok()) - .unwrap_or_else(|| a.get().to_string()) - }); - let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); - let script_path_str = script_path.as_deref().unwrap_or("None"); - - let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); - - let bold_format = match format { - MessageFormat::Slack => "*{}*", - MessageFormat::Teams => "**{}**", - }; - - let mut message_str = format!( - "A workflow has been suspended and is waiting for approval:\n\n\ - {}: {created_by}\n\n\ - {}: {created_at_formatted}\n\n\ - {}: {script_path_str}\n\n\ - {}:\n```\n{args_str}\n```\n\n\ - {}: {parent_job_id_str}\n\n", - bold_format.replace("{}", "Created by"), - bold_format.replace("{}", "Created at"), - bold_format.replace("{}", "Script path"), - bold_format.replace("{}", "Args"), - bold_format.replace("{}", "Flow ID") - ); - - // Append custom message if provided - if let Some(msg) = message { - message_str.push_str(msg); - } - - tracing::debug!("Schema: {:#?}", schema); - - Ok(ApprovalFormDetails { message_str, urls, schema }) + Ok(ApprovalStep { + created_by, + created_at, + script_path, + parent_job_id, + args, + suspend: module.and_then(|m| m.suspend.clone()), + }) } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 7bc995844d..432af1bef0 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -57,7 +57,7 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE}, - auth::TOKEN_PREFIX_LEN, + auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, cache::{self, future::FutureCachedExt}, db::{DbWithOptAuthed, UserDB}, error::{to_anyhow, Error, JsonResult, Result}, @@ -69,8 +69,8 @@ use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - strip_json_nul, Pagination, RunnableKind, StripPath, + http_get_from_hub, not_found_if_none, paginate, paginate_optional, + query_elems_from_hub, require_admin, strip_json_nul, Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -170,6 +170,7 @@ pub fn unauthed_service() -> Router { ) .route("/load_csv_preview/{*path}", get(app_load_csv_preview)) .route("/public_app/{secret}", get(get_public_app_by_secret)) + .route("/guest_entry/{secret}", get(get_guest_entry)) .route("/embed_token/{secret}", get(get_app_embed_token)) .route("/public_resource/{*path}", get(get_public_resource)) .route("/get_data/v/{*id}", get(get_raw_app_data)) @@ -273,6 +274,12 @@ pub struct AppHistory { pub version: i64, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + /// Who deployed this version, and when — the diff's version picker names them so + /// a reader can tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub created_at: Option>, } #[derive(Deserialize)] @@ -288,6 +295,15 @@ pub type AllowUserResources = Vec; #[serde(rename_all = "lowercase")] pub enum ExecutionMode { Anonymous, + /// Login required, workspace membership not: anyone the instance's identity + /// provider authenticates may open the app, and the runnables execute as the + /// publisher exactly as in [`ExecutionMode::Publisher`]. Such a viewer holds a + /// guest session: an identity with no account at all (no `password` row, no `usr` + /// row anywhere), which is what keeps it off every row-based seat counter; what a + /// guest costs instead is the allowance in `windmill_common::workspaces`. Honored + /// only where `workspace_settings.guest_access_enabled` is on, re-read at the auth + /// door on every guest request. + Guest, /// Default for a policy that omits `execution_mode`. It MUST stay a mode /// that requires an authenticated viewer: an omitted field must never be /// able to publish an app anonymously (publicly executable). @@ -296,6 +312,159 @@ pub enum ExecutionMode { Viewer, } +impl ExecutionMode { + /// The serialized form, matching this enum's `rename_all = "lowercase"`. + pub fn as_str(&self) -> &'static str { + match self { + ExecutionMode::Anonymous => "anonymous", + ExecutionMode::Guest => "guest", + ExecutionMode::Publisher => "publisher", + ExecutionMode::Viewer => "viewer", + } + } +} + +/// The protection rule gating a *transition into* `mode`, if any. Anonymous and +/// guest each widen who may open an app past the workspace's own members, so each +/// carries its own rule; the two member-only modes are ungated. +fn deployment_rule_for_mode(mode: ExecutionMode) -> Option { + match mode { + ExecutionMode::Anonymous => Some(ProtectionRuleKind::RestrictAnonymousAppDeployment), + ExecutionMode::Guest => Some(ProtectionRuleKind::RestrictGuestAppDeployment), + ExecutionMode::Publisher | ExecutionMode::Viewer => None, + } +} + +/// A guest session is scoped to its app by path, so an app whose path the scope +/// grammar cannot hold as one literal (`is_scope_literal_path`) can never admit a +/// guest; refuse the mode at deploy time rather than advertise an app nobody enters. +/// `path` is where the app ends up: on a rename, the destination. +fn refuse_unscopable_guest_app(path: &str, mode: ExecutionMode) -> Result<()> { + if matches!(mode, ExecutionMode::Guest) && !windmill_common::auth::is_scope_literal_path(path) { + return Err(Error::BadRequest(format!( + "app {path} cannot be set to Guests: a path with `:`, `,` or `*`, or a leading `/`, \ + cannot be scoped" + ))); + } + Ok(()) +} + +/// Refuse *widening* an app into guests where the deployment has none +/// (`instance_supports_guests`). Only the transition is refused, like the protection +/// rule below it: an app already stored in the mode — deployed before the instance +/// became a cloud one, or pushed by git-sync — keeps deploying, and keeps being inert, +/// since every guest gate refuses it anyway. `deployed_mode` is what the app is stored +/// as, `None` when it is being created. +fn refuse_guest_mode_where_unavailable( + path: &str, + mode: ExecutionMode, + deployed_mode: Option, +) -> Result<()> { + if !matches!(mode, ExecutionMode::Guest) + || deployed_mode == Some(ExecutionMode::Guest) + || windmill_common::workspaces::instance_supports_guests() + { + return Ok(()); + } + Err(Error::BadRequest(format!( + "app {path} cannot be set to Guests: {}", + windmill_common::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))) +} + +/// Gate a viewer on the app's `execution_mode`, as far as can be decided without an +/// ACL probe. `Ok(true)` means already authorized — anonymous admits anyone, guest +/// admits anyone signed in; `Ok(false)` means the caller is a member and still owes +/// the read-access check its caller performs. +/// +/// A guest is authorized by its token's scope and never by an ACL probe: it holds no +/// `usr` row, so RLS finds nothing for it and every guest would read as having no +/// access. That scope is also what keeps a guest session to the one app it was minted +/// for, even though the mode itself admits anyone signed in. The workspace's guest +/// switch is not checked here: `AuthCache` enforces it for every guest request. +pub fn authorize_non_member_viewer( + mode: ExecutionMode, + app_path: &str, + opt_authed: &Option, +) -> Result { + if matches!(mode, ExecutionMode::Anonymous) { + return Ok(true); + } + let Some(authed) = opt_authed.as_ref() else { + return Err(Error::NotAuthorized( + "App visibility does not allow public access and you are not logged in".to_string(), + )); + }; + let is_guest = windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()); + if matches!(mode, ExecutionMode::Guest) { + if is_guest { + check_scopes(authed, || format!("apps:read:{}", app_path))?; + } + return Ok(true); + } + if is_guest { + return Err(Error::PermissionDenied(format!( + "app {app_path} is not open to guests" + ))); + } + Ok(false) +} + +/// Confines a guest to its app once the app's mode is known; a no-op for every other +/// caller. An anonymous app is open to anyone, so the guest stays the caller there, +/// as itself: the run and the reads that follow it (job results, S3 provenance) must +/// carry one identity. Anywhere else a mismatch is refused. +fn guest_caller_for_mode( + opt_authed: Option, + mode: ExecutionMode, + app_path: &str, +) -> Result> { + let Some(authed) = opt_authed.as_ref() else { + return Ok(None); + }; + if !windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) + || matches!(mode, ExecutionMode::Anonymous) + { + return Ok(opt_authed); + } + check_scopes(authed, || format!("apps:run:{app_path}")) + .or_else(|_| check_scopes(authed, || format!("apps:read:{app_path}")))?; + Ok(opt_authed) +} + +/// [`authorize_non_member_viewer`] plus the member read-access probe, for the +/// entry points that address an app by id. +async fn authorize_app_viewer( + mode: ExecutionMode, + app_path: &str, + app_id: i64, + w_id: &str, + user_db: &UserDB, + opt_authed: &Option, +) -> Result<()> { + if authorize_non_member_viewer(mode, app_path, opt_authed)? { + return Ok(()); + } + let authed = opt_authed + .as_ref() + .ok_or_else(|| Error::internal_err("authorize_app_viewer: unauthenticated".to_string()))?; + let mut tx = user_db.clone().begin(authed).await?; + let is_visible = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)", + app_id, + w_id + ) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + if !is_visible.unwrap_or(false) { + return Err(Error::NotAuthorized( + "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(), + )); + } + Ok(()) +} + #[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct PolicyTriggerableInputs { static_inputs: StaticFields, @@ -328,6 +497,13 @@ pub struct S3Key { #[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Policy { pub on_behalf_of: Option, + /// The address `on_behalf_of` resolves to. Every write stores what the principal resolves + /// to, so it is not taken from the request except when a client names only the address — + /// which is how a cross-workspace deploy carries an identity — and it is rejected when the + /// two disagree. Optional: a policy without it executes by deriving from the principal, so + /// removing it is a change of default rather than of behavior — see + /// `docs/app-policy-email-removal.md`. + #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, //paths: // - script/ @@ -397,6 +573,17 @@ pub struct CreateApp { pub skip_draft_deletion: Option, } +/// What a deploy of an existing app answers with. `version` is the one this call wrote, +/// which is what an editor pins as the fork base of the draft it starts next: reading the +/// head back afterwards cannot tell it from a deploy that landed beside it. A +/// metadata-only update writes none and reports the head it kept. +#[derive(Serialize)] +pub struct AppDeployed { + /// Where the app now lives, which differs from the request path on a rename. + pub path: String, + pub version: i64, +} + #[derive(Serialize, Deserialize)] pub struct EditApp { pub path: Option, @@ -491,7 +678,8 @@ async fn list_apps( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app')) as draft_users", + WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app') \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(app.workspace_id, app.path) as inherited_labels", ]) .left() @@ -1049,18 +1237,42 @@ async fn get_app_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", &path))?; + // Unasked-for, this listing stays whole: the deployment-history panel reads it + // without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; + // Newest first in deployed order, which is a version's position in `app.versions` and + // not its `created_at`: the latter is the deploying transaction's start time, so two + // that overlap can carry it in the opposite order from the one they landed in. A row + // outside the array never sat in that sequence, so it sorts after the ones that did. + // Paging happens before the metadata joins, so a page costs its own rows. let query_result = sqlx::query!( - "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg - FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg, + av.created_by as created_by, av.created_at as created_at + FROM app a + JOIN LATERAL ( + SELECT av2.id, COALESCE(v.ord, 0) AS ord + FROM app_version av2 + LEFT JOIN unnest(a.versions) WITH ORDINALITY AS v(id, ord) ON v.id = av2.id + WHERE av2.app_id = a.id + ORDER BY ord DESC, av2.id DESC + LIMIT $3 OFFSET $4 + ) page ON TRUE + JOIN app_version av ON av.id = page.id + LEFT JOIN deployment_metadata dm ON av.id = dm.app_version WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY created_at DESC", + ORDER BY page.ord DESC", w_id, path, - ).fetch_all(&mut *tx).await?; + per_page, + offset, + ) + .fetch_all(&mut *tx) + .await?; tx.commit().await?; let result: Vec = query_result @@ -1069,6 +1281,8 @@ async fn get_app_history( app_id: row.app_id, version: row.version_id, deployment_msg: row.deployment_msg, + created_by: Some(row.created_by), + created_at: Some(row.created_at), }) .collect(); return Ok(Json(result)); @@ -1082,14 +1296,22 @@ async fn get_latest_version( let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; + // The head is the tail of `app.versions` — the version the runtime serves. Deploys + // append to it under the app row's lock, whereas `app_version.created_at` is the + // deploying transaction's start time, so two that overlap can carry it in either + // order and the newest timestamp is then not the one that landed last. let row = sqlx::query!( - "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg - FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version - WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY created_at DESC", + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg, + av.created_by as created_by, av.created_at as created_at + FROM app a JOIN app_version av + ON av.id = a.versions[array_upper(a.versions, 1)] AND av.app_id = a.id + LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + WHERE a.workspace_id = $1 AND a.path = $2", w_id, path, - ).fetch_optional(&mut *tx).await?; + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; if let Some(row) = row { @@ -1097,6 +1319,8 @@ async fn get_latest_version( app_id: row.app_id, version: row.version_id, deployment_msg: row.deployment_msg, + created_by: Some(row.created_by), + created_at: Some(row.created_at), }; return Ok(Json(Some(result))); @@ -1217,29 +1441,15 @@ async fn get_public_app_by_secret( let policy = serde_json::from_str::(app.policy.0.get()).map_err(to_anyhow)?; - if !matches!(policy.execution_mode(), ExecutionMode::Anonymous) { - if opt_authed.is_none() { - return Err(Error::NotAuthorized( - "App visibility does not allow public access and you are not logged in".to_string(), - )); - } else { - let authed = opt_authed.unwrap(); - let mut tx = user_db.begin(&authed).await?; - let is_visible = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)", - id, - &w_id - ) - .fetch_one(&mut *tx) - .await?; - tx.commit().await?; - if !is_visible.unwrap_or(false) { - return Err(Error::NotAuthorized( - "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(), - )); - } - } - } + authorize_app_viewer( + policy.execution_mode(), + &app.path, + id, + &w_id, + &user_db, + &opt_authed, + ) + .await?; // Compute bundle_secret for raw apps if app.raw_app { @@ -1283,12 +1493,14 @@ const APP_EMBED_TOKEN_VALIDITY_HOURS: i64 = 12; /// Scopes an app author may declare in `Policy::frontend_sdk_scopes`. No `apps:*` /// scope, so the token cannot reach the mint endpoints and renew itself; the /// `raw_app_sdk` sentinel narrows the rest (see `scopes.rs`). -pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 5] = [ +pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 7] = [ "jobs:run", "jobs:read", "users:read", "resources:read", "variables:read", + "flow_conversations:read", + "flow_conversations:write", ]; /// Reject a policy declaring frontend SDK scopes outside the curated list. @@ -1355,9 +1567,21 @@ async fn mint_raw_app_sdk_token( ensure_scopes_within_caller(authed, Some(scopes))?; let mut scopes = scopes.to_vec(); scopes.push(windmill_api_auth::scopes::RAW_APP_SDK_SENTINEL.to_string()); - let expiration = chrono::Utc::now() + chrono::Duration::hours(APP_EMBED_TOKEN_VALIDITY_HOURS); + let requested_exp = + chrono::Utc::now() + chrono::Duration::hours(APP_EMBED_TOKEN_VALIDITY_HOURS); + let (label, expiration) = + match guest_derived_token_constraints(db, authed, requested_exp).await? { + Some((label, exp)) => { + scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string()); + (label, exp) + } + None => ( + format!("{RAW_APP_SDK_TOKEN_LABEL_PREFIX}{app_path}"), + requested_exp, + ), + }; let token_config = NewToken::new( - Some(format!("sdk_app:{app_path}")), + Some(label), Some(expiration), None, Some(scopes), @@ -1372,6 +1596,43 @@ async fn mint_raw_app_sdk_token( Ok((token, expiration)) } +/// Label and expiry a token minted *by* a guest session must carry, or `None` for a +/// non-guest minter. The label is what lets it resolve; the caller pushes the `guest` +/// sentinel so every guest control still applies; the expiry is capped at the parent's, +/// since that expiry is a guest's only revocation short of logging out. +async fn guest_derived_token_constraints( + db: &DB, + authed: &ApiAuthed, + requested: chrono::DateTime, +) -> Result)>> { + if !windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) { + return Ok(None); + } + // A guest JWT carries its own expiry and has no token row to look up; a signed-in + // guest session is a row found by prefix (MIN is the conservative side of a + // theoretical prefix collision). Either way the derived token caps on it, never on + // a fresh interval. + let parent_exp = if let Some(exp) = authed.credential_expiry { + exp + } else { + let parent: Option>> = sqlx::query_scalar( + "SELECT MIN(expiration) FROM token WHERE token_prefix = $1 AND email = $2 AND label = $3", + ) + .bind(authed.token_prefix.as_deref().unwrap_or("")) + .bind(&authed.email) + .bind(windmill_common::auth::GUEST_SESSION_LABEL) + .fetch_optional(db) + .await?; + parent.flatten().ok_or_else(|| { + Error::NotAuthorized("guest session not found or has no expiry".to_string()) + })? + }; + Ok(Some(( + windmill_common::auth::GUEST_SESSION_LABEL.to_string(), + requested.min(parent_exp), + ))) +} + /// Shared tail of the three embed-token endpoints: which credential the viewer /// gets. Sandboxed low-code gets the embed token; a sandboxed raw app declaring /// `frontend_sdk_scopes` gets the SDK token once `sdk_consent` is set — the @@ -1399,7 +1660,22 @@ pub async fn build_embed_token_response( && opt_authed.is_some() && !policy.frontend_sdk_scopes.is_empty() { - Some(policy.frontend_sdk_scopes.clone()) + // An SDK token runs as the viewer, and a guest's session is the ceiling on what + // it may delegate — the mint enforces that. Advertise only what a guest can + // actually be granted, so the consent prompt never promises a scope the mint + // would then refuse. + let declared = policy.frontend_sdk_scopes.clone(); + let offered = match opt_authed { + Some(a) if windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref()) => { + let held = a.scopes.as_deref().unwrap_or_default(); + declared + .into_iter() + .filter(|sc| held.iter().any(|h| h == sc)) + .collect::>() + } + _ => declared, + }; + (!offered.is_empty()).then_some(offered) } else { None }; @@ -1555,9 +1831,13 @@ pub async fn mint_app_embed_token( "App embed tokens cannot mint or renew embed tokens".to_string(), )); } - let expiration = + let requested_exp = chrono::Utc::now() + chrono::Duration::hours(APP_EMBED_TOKEN_VALIDITY_HOURS); - let mut scopes: Vec = APP_EMBED_SCOPES.iter().map(|s| s.to_string()).collect(); + let mut scopes: Vec = APP_EMBED_SCOPES + .iter() + .filter(|s| **s != windmill_api_auth::scopes::APP_EMBED_SENTINEL) + .map(|s| s.to_string()) + .collect(); // Path-scoped read so the app can fetch its OWN definition (apps/get/p, // which the in-workspace sandboxed viewer uses) — but no other app's. The // public viewer fetches via apps_u/public_app and doesn't rely on this. @@ -1568,10 +1848,25 @@ pub async fn mint_app_embed_token( scopes.push(format!("apps:run:{app_path}")); // A scope-restricted caller token must not bootstrap a broader-scoped // embed token (`create_token_internal` deliberately does not check this - // itself). No-op for unscoped sessions — the normal embed flow. + // itself). Checked on the real scopes only: a sentinel is a one-part string + // that `ScopeDefinition::from_scope_string` rejects, so leaving it in the + // requested set makes this fail outright for any scoped caller — which a + // guest session is. `mint_raw_app_sdk_token` has the same shape. ensure_scopes_within_caller(authed, Some(&scopes))?; + scopes.push(windmill_api_auth::scopes::APP_EMBED_SENTINEL.to_string()); + let (label, expiration) = + match guest_derived_token_constraints(db, authed, requested_exp).await? { + Some((label, exp)) => { + scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string()); + (label, exp) + } + None => ( + format!("{APP_EMBED_TOKEN_LABEL_PREFIX}{app_path}"), + requested_exp, + ), + }; let token_config = NewToken::new( - Some(format!("embed_app:{app_path}")), + Some(label), Some(expiration), None, Some(scopes), @@ -1600,6 +1895,40 @@ pub async fn mint_app_embed_token( }) } +#[derive(Serialize)] +pub struct GuestEntry { + /// The workspace and app path to name when starting a guest sign-in. The + /// workspace is redundant on the secret route and load-bearing on the custom-path + /// one, which may not carry it in its URL. + pub workspace_id: String, + pub app_path: String, +} + +/// Whether the app behind this share secret admits guests, and under what path. +/// +/// Unauthenticated on purpose: it is what a signed-out visitor reads to learn that +/// signing in would get them in. It discloses only the app's path, to a caller who +/// already holds the share secret — the secret is the capability here. A 404 when the +/// app is not open to guests, so it says nothing about apps that are not. +async fn get_guest_entry( + Extension(db): Extension, + Path((w_id, secret)): Path<(String, String)>, +) -> JsonResult { + let id = get_id_from_secret(&db, &w_id, secret, None).await?; + let app = sqlx::query!( + "SELECT path FROM app WHERE id = $1 AND workspace_id = $2", + id, + &w_id + ) + .fetch_optional(&db) + .await?; + let app = not_found_if_none(app, "App", id.to_string())?; + if !windmill_common::workspaces::guest_app_admits(&db, &w_id, &app.path).await? { + return Err(Error::NotFound("App is not open to guests".to_string())); + } + Ok(Json(GuestEntry { workspace_id: w_id, app_path: app.path })) +} + /// Issue an embed token for a public app addressed by its (secret) share id. /// Mirrors the access check in [`get_public_app_by_secret`]: anonymous apps are /// reachable without auth, otherwise the caller must be logged in and have read @@ -1645,29 +1974,18 @@ async fn get_app_embed_token( let authed_for_token = if policy.anonymous_execution { // Anonymous app: still mint a scoped token if the viewer happens to be - // logged in (so the app sees their identity), otherwise stay anonymous. - opt_authed + // logged in (so the app sees their identity), otherwise stay anonymous. A + // guest's session names another app and cannot contain this one's scopes, + // so it renders anonymously here rather than being refused. + opt_authed.filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) } else { - let authed = opt_authed.ok_or_else(|| { - Error::NotAuthorized( - "App visibility does not allow public access and you are not logged in".to_string(), - ) - })?; - let mut tx = user_db.begin(&authed).await?; - let is_visible = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM app WHERE id = $1 AND workspace_id = $2)", - id, - &w_id - ) - .fetch_one(&mut *tx) - .await?; - tx.commit().await?; - if !is_visible.unwrap_or(false) { - return Err(Error::NotAuthorized( - "App visibility does not allow public access and you are logged in but you have no read-access to that app".to_string(), - )); - } - Some(authed) + let mode = if policy.guest_execution { + ExecutionMode::Guest + } else { + ExecutionMode::Publisher + }; + authorize_app_viewer(mode, &app.path, id, &w_id, &user_db, &opt_authed).await?; + opt_authed }; let resp = build_embed_token_response( @@ -1693,6 +2011,9 @@ async fn get_app_embed_token( /// strictest access interpretation. pub struct EmbedPolicyView { pub anonymous_execution: bool, + /// Open to anyone the identity provider authenticates. Like + /// `anonymous_execution`, an unknown mode reads as `false` — the strict side. + pub guest_execution: bool, pub sandbox: bool, /// Raw apps: author-declared scopes for the frontend SDK token; empty when /// the app doesn't use the frontend SDK (non-string entries are ignored). @@ -1703,6 +2024,7 @@ pub fn parse_embed_policy(policy_str: &str) -> Result { let v: serde_json::Value = serde_json::from_str(policy_str).map_err(to_anyhow)?; Ok(EmbedPolicyView { anonymous_execution: v.get("execution_mode").and_then(|m| m.as_str()) == Some("anonymous"), + guest_execution: v.get("execution_mode").and_then(|m| m.as_str()) == Some("guest"), sandbox: v.get("sandbox").and_then(|b| b.as_bool()).unwrap_or(false), frontend_sdk_scopes: v .get("frontend_sdk_scopes") @@ -2207,31 +2529,37 @@ async fn create_app_internal<'a>( } // Resolve the on-behalf-of defaults on the (non-RLS) pool *before* opening // the RLS transaction below: doing these lookups mid-transaction would hold - // a second simultaneous connection while `tx` is still checked out. + // a second simultaneous connection while `tx` is still checked out. The race this + // leaves with a concurrent rename or removal, including a freed username later + // rebinding the stored principal, is known and accepted: see `resolve_on_behalf_of`. let should_preserve = app.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed) - && app.policy.on_behalf_of.is_some(); + && (app.policy.on_behalf_of.is_some() || app.policy.on_behalf_of_email.is_some()); - if !should_preserve { + let mut preserved_on_behalf_of: Option = None; + if should_preserve { + app.policy.on_behalf_of = windmill_common::resolve_on_behalf_of( + app.policy.on_behalf_of_email.as_deref(), + app.policy.on_behalf_of.as_deref(), + true, + &authed, + w_id, + &db, + ) + .await?; + } else { let folder_default = if windmill_common::can_preserve_on_behalf_of(&authed) { windmill_common::folders::resolve_folder_default_permissioned_as(&db, w_id, &app.path) .await? } else { None }; - if let Some(default_permissioned_as) = folder_default { - let default_email = windmill_common::users::get_email_from_permissioned_as( - &default_permissioned_as, - w_id, - &db, - ) - .await?; - app.policy.on_behalf_of = Some(default_permissioned_as); - app.policy.on_behalf_of_email = Some(default_email); - } else { - app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - app.policy.on_behalf_of_email = Some(authed.email.clone()); - } + app.policy.on_behalf_of = + Some(folder_default.unwrap_or_else(|| username_to_permissioned_as(&authed.username))); + } + app.policy.on_behalf_of_email = stored_on_behalf_of_email(&app.policy, w_id, &db).await?; + if should_preserve { + preserved_on_behalf_of = audited_on_behalf_of(&app.policy, &authed); } // Reject a forged superadmin run identity in the (possibly preserved) policy. @@ -2286,10 +2614,12 @@ async fn create_app_internal<'a>( // Pin the mode the app is created under, so the stored policy states one // even when the caller did not. app.policy.set_execution_mode(app.policy.execution_mode()); - if matches!(app.policy.execution_mode(), ExecutionMode::Anonymous) { + refuse_unscopable_guest_app(&app.path, app.policy.execution_mode())?; + refuse_guest_mode_where_unavailable(&app.path, app.policy.execution_mode(), None)?; + if let Some(rule) = deployment_rule_for_mode(app.policy.execution_mode()) { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, - &ProtectionRuleKind::RestrictAnonymousAppDeployment, + &rule, &authed.username, &authed.groups, authed.is_admin, @@ -2314,6 +2644,14 @@ async fn create_app_internal<'a>( .execute(&mut *tx) .await?; } + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::App, UserDraftItemKind::RawApp], + &app.path, + None, + ) + .await?; let id = sqlx::query_scalar!( "INSERT INTO app (workspace_id, path, summary, policy, versions, custom_path, labels) @@ -2365,21 +2703,17 @@ async fn create_app_internal<'a>( None, ) .await?; - if should_preserve { - if let Some(ref obo_email) = app.policy.on_behalf_of_email { - if obo_email != &authed.email { - audit_log( - &mut *tx, - &authed, - "apps.on_behalf_of", - ActionKind::Create, - w_id, - Some(&app.path), - Some([("on_behalf_of", obo_email.as_str()), ("action", "create")].into()), - ) - .await?; - } - } + if let Some(ref obo_email) = preserved_on_behalf_of { + audit_log( + &mut *tx, + &authed, + "apps.on_behalf_of", + ActionKind::Create, + w_id, + Some(&app.path), + Some([("on_behalf_of", obo_email.as_str()), ("action", "create")].into()), + ) + .await?; } let mut args: HashMap> = HashMap::new(); if let Some(dm) = &app.deployment_message { @@ -2674,7 +3008,7 @@ async fn update_app( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -2713,7 +3047,7 @@ async fn update_app( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } /// Deploy a raw app from its sources, compiling them on a worker. `update_raw` @@ -2727,7 +3061,7 @@ async fn update_app_raw_source( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -2825,7 +3159,7 @@ async fn update_app_raw_source( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } /// Whether the caller may create an app at `path` — asked of the database rather @@ -3051,7 +3385,7 @@ async fn update_app_raw<'a>( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, multipart: Multipart, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3099,7 +3433,7 @@ async fn update_app_raw<'a>( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } // async fn create_app_internal<'a>( // authed: ApiAuthed, @@ -3136,19 +3470,33 @@ async fn update_app_internal<'a>( } } - // Reject a forged superadmin run identity in a preserved policy. Mirror the - // `should_preserve` gate below (only a preserved value is caller-controlled; - // otherwise the policy is rewritten to the deployer's own identity) and run - // it on the non-RLS pool before the transaction to avoid a second connection. - if let Some(npolicy) = ns.policy.as_ref() { + // Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason + // `create_app` states, with the same known, accepted rename race (see + // `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution + // identity; a source deploy that sent none claims nothing, so whoever the app already runs as + // stays. + let mut preserved_on_behalf_of: Option = None; + if let Some(npolicy) = ns.policy.as_mut() { let should_preserve = ns.preserve_on_behalf_of.unwrap_or(false) && windmill_common::can_preserve_on_behalf_of(&authed) - && npolicy.on_behalf_of.is_some(); + && (npolicy.on_behalf_of.is_some() || npolicy.on_behalf_of_email.is_some()); + if should_preserve { - windmill_common::auth::validate_on_behalf_of( - npolicy.on_behalf_of.as_deref(), + npolicy.on_behalf_of = windmill_common::resolve_on_behalf_of( npolicy.on_behalf_of_email.as_deref(), - )?; + npolicy.on_behalf_of.as_deref(), + true, + &authed, + w_id, + &db, + ) + .await?; + } else { + npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); + } + npolicy.on_behalf_of_email = stored_on_behalf_of_email(npolicy, w_id, &db).await?; + if should_preserve { + preserved_on_behalf_of = audited_on_behalf_of(npolicy, &authed); } } @@ -3181,7 +3529,6 @@ async fn update_app_internal<'a>( reject_kind_change(path, raw_app, deployed_raw_app)?; } - let mut preserved_on_behalf_of: Option = None; let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() @@ -3200,6 +3547,28 @@ async fn update_app_internal<'a>( if npath != path { require_owner_of_path(&authed, path)?; + // The destination is what a guest session would be scoped to. A rename + // that carries no policy keeps the deployed mode, read under the row + // lock so a policy update landing alongside cannot slip a guest app + // onto a path it cannot be scoped to. + let mode = match ns.policy.as_ref().and_then(|p| p.stated_execution_mode()) { + Some(mode) => mode, + None => sqlx::query_scalar::<_, Option>( + "SELECT policy->>'execution_mode' FROM app + WHERE path = $1 AND workspace_id = $2 FOR UPDATE", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await? + .flatten() + .and_then(|m| { + serde_json::from_value::(serde_json::Value::String(m)).ok() + }) + .unwrap_or_default(), + }; + refuse_unscopable_guest_app(npath, mode)?; + let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)", npath, @@ -3307,21 +3676,33 @@ async fn update_app_internal<'a>( .unwrap_or_default(), ); } - if matches!(npolicy.execution_mode(), ExecutionMode::Anonymous) && !authed.is_admin { - // Restricted users may keep deploying an app that is already - // public, but flipping an app to anonymous (public) access is - // gated by the RestrictAnonymousAppDeployment protection rule. - // An unreadable deployed policy reads as not-anonymous, the - // strict direction. - let already_anonymous = deployed + refuse_unscopable_guest_app( + ns.path.as_deref().unwrap_or(path), + npolicy.execution_mode(), + )?; + // An unreadable deployed policy reads as not already-in-mode, the strict + // direction, as for the protection rule below. + refuse_guest_mode_where_unavailable( + ns.path.as_deref().unwrap_or(path), + npolicy.execution_mode(), + deployed_policy.as_ref().map(|d| d.execution_mode()), + )?; + if let Some(rule) = + deployment_rule_for_mode(npolicy.execution_mode()).filter(|_| !authed.is_admin) + { + // Restricted users may keep deploying an app that is already open + // to this audience, but widening one is gated by the matching + // protection rule. An unreadable deployed policy reads as not + // already-widened, the strict direction. + let already_in_mode = deployed .as_ref() .and_then(|p| p.get("execution_mode")) .and_then(|m| m.as_str()) - == Some("anonymous"); - if !already_anonymous { + == Some(npolicy.execution_mode().as_str()); + if !already_in_mode { if let RuleCheckResult::Blocked(msg) = check_user_against_rule( w_id, - &ProtectionRuleKind::RestrictAnonymousAppDeployment, + &rule, &authed.username, &authed.groups, authed.is_admin, @@ -3333,23 +3714,6 @@ async fn update_app_internal<'a>( } } } - let should_preserve = ns.preserve_on_behalf_of.unwrap_or(false) - && windmill_common::can_preserve_on_behalf_of(&authed) - && npolicy.on_behalf_of.is_some(); - - if should_preserve { - if let Some(ref obo_email) = npolicy.on_behalf_of_email { - if obo_email != &authed.email { - preserved_on_behalf_of = Some(obo_email.clone()); - } - } - } else if caller_sent_policy { - // Submitting a policy is how a deployer claims the app's - // execution identity. A source deploy that sent none is not - // claiming anything, so whoever the app already runs as stays. - npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - npolicy.on_behalf_of_email = Some(authed.email.clone()); - } sqlb.set( "policy", quote(serde_json::to_string(&json!(npolicy)).map_err(|e| { @@ -3446,6 +3810,19 @@ async fn update_app_internal<'a>( .execute(&mut *tx) .await?; } + if npath != path { + // Everything left at the old path is a draft this deploy didn't consume + // — teammates' rows, and the deployer's own when the caller asked us to + // keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::App, UserDraftItemKind::RawApp], + path, + &npath, + ) + .await?; + } audit_log( &mut *tx, &authed, @@ -3556,20 +3933,59 @@ fn digest(code: &str) -> String { format!("rawscript/{:x}", result) } +/// Canonical `runnable_path` for a run-mode no-id inline app component: +/// `/` — byte-for-byte what the runtime frontend sends. It is +/// the relative-import base, so the caller must not steer it: reject a `component` +/// that isn't a single non-empty path segment (empty, a separator, or `.`/`..` +/// would walk the base out of the app, and a bare `rawscript/` policy key +/// does not pin the component). +fn inline_run_path(app_path: &str, component: &str) -> Result { + if component.is_empty() + || component.contains('/') + || component.contains('\\') + || component == "." + || component == ".." + { + return Err(Error::BadRequest( + "component id must be a single non-empty path segment".to_string(), + )); + } + Ok(format!("{app_path}/{component}")) +} + async fn get_on_behalf_details_from_policy_and_authed( policy: &Policy, opt_authed: &Option, + w_id: &str, + db: &DB, ) -> Result<(String, String, String)> { + // A guest acts only through an app open to guests — or to everyone. A members-only + // mode means the policy changed after the session was issued. Decided here, in the + // one resolver every on-behalf path (runs, S3 reads, uploads) goes through. + if opt_authed + .as_ref() + .is_some_and(|a| windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + && !matches!( + policy.execution_mode(), + ExecutionMode::Guest | ExecutionMode::Anonymous + ) + { + return Err(Error::PermissionDenied( + "this app is not open to guests".to_string(), + )); + } let (username, permissioned_as, email) = match policy.execution_mode() { ExecutionMode::Anonymous => { let username = opt_authed .as_ref() .map(|a| a.username.clone()) .unwrap_or_else(|| "anonymous".to_string()); - let (permissioned_as, email) = get_on_behalf_of(&policy)?; + let (permissioned_as, email) = get_on_behalf_of(&policy, w_id, db).await?; (username, permissioned_as, email) } - ExecutionMode::Publisher => { + // Guest runs as the publisher exactly as Publisher does; the two differ only + // in who is let through the door, which is settled before we get here. + ExecutionMode::Publisher | ExecutionMode::Guest => { let username = opt_authed .as_ref() .map(|a| a.username.clone()) @@ -3578,7 +3994,7 @@ async fn get_on_behalf_details_from_policy_and_authed( "publisher execution mode requires authentication".to_string(), ) })?; - let (permissioned_as, email) = get_on_behalf_of(&policy)?; + let (permissioned_as, email) = get_on_behalf_of(&policy, w_id, db).await?; (username, permissioned_as, email) } ExecutionMode::Viewer => { @@ -3654,8 +4070,12 @@ async fn execute_component( // Authorize before touching the payload: the route layer is resource-blind, so a // path-scoped caller (app embed token, or a picker-minted `apps:run|write:`) // is confined to its own app only here. No-op for unscoped callers; anonymous ones - // are policy-gated below. - if let Some(authed) = opt_authed.as_ref() { + // are policy-gated below, and a guest's confinement waits for the app's mode + // (`guest_caller_for_mode`). + if let Some(authed) = opt_authed + .as_ref() + .filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + { check_scopes(authed, || format!("apps:run:{}", path))?; } // Only honor temp_script_refs for the inline-script preview path: @@ -3872,8 +4292,16 @@ async fn execute_component( } }; - // Check rate limit for anonymous (public) executions - if matches!(policy.execution_mode(), ExecutionMode::Anonymous) && opt_authed.is_none() { + // Rate limit for executions by callers the workspace does not know: anonymous + // viewers, and guests — on an instance whose provider accepts any consumer + // account, "anyone the IdP authenticates" is close to the anonymous population, + // and each run costs a job as the publisher. + let is_guest_caller = opt_authed + .as_ref() + .is_some_and(|a| windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())); + if (matches!(policy.execution_mode(), ExecutionMode::Anonymous) && opt_authed.is_none()) + || is_guest_caller + { if let Some(limit) = crate::workspaces::get_public_app_rate_limit(&db, &w_id).await? { if limit > 0 { crate::public_app_rate_limit::check_and_increment(&w_id, limit)?; @@ -3881,6 +4309,8 @@ async fn execute_component( } } + let opt_authed = guest_caller_for_mode(opt_authed, policy.execution_mode(), path)?; + // Execution is publisher and an user is authenticated: check if the user is authorized to // execute the app. if let (ExecutionMode::Publisher, Some(authed)) = (policy.execution_mode(), opt_authed.as_ref()) @@ -3922,11 +4352,19 @@ async fn execute_component( } let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); + // `_MODULES` and `_TEMP_SCRIPT_REFS` are server-injected control keys (into + // `extra`) that the worker reads back for a `Preview` job — which an inline run + // is. A caller supplying them in `args` would inject module content/locks or + // redirect relative-import resolution, unpinned, as the app identity. Drop them; + // legitimate values ride in `extra`, never the request `args`. + payload.args.remove("_MODULES"); + payload.args.remove("_TEMP_SCRIPT_REFS"); + let (mut args, job_id) = build_args( policy, policy_triggerables, @@ -3946,11 +4384,11 @@ async fn execute_component( } } - let is_flow = payload + let flow_path = payload .path - .as_ref() - .map(|p| p.starts_with("flow/")) - .unwrap_or(false); + .as_deref() + .and_then(|path| path.strip_prefix("flow/")) + .map(str::to_string); // Tag for inline-script jobs is read from the deployed policy in run mode; // only preview mode (editor) honors the client-supplied tag. This applies to @@ -3964,6 +4402,7 @@ async fn execute_component( } .filter(|t| !t.is_empty()) }; + let component = payload.component.clone(); let (job_payload, tag, _runnable_on_behalf_of) = match (payload.path, payload.raw_code, payload.id) { // flow or script: @@ -3974,6 +4413,29 @@ async fn execute_component( // `app_script` table (legacy `rawscript/`-keyed triggerables). (None, Some(raw_code), None) => { let tag = resolved_inline_tag(raw_code.tag.clone()); + let raw_code = if is_preview { + // Preview (editor / `wmill app dev`): the caller runs their own + // code, like `/jobs/run/preview` — honored verbatim. + raw_code + } else { + // Run mode. Legacy back-compat only: current deploys assign an + // `app_script` id (reduce_app) and take the `Some(id)` arm; + // drop this branch once id-less deployed apps are gone. + // + // Only `content` is pinned (`rawscript/`), so keep just + // that plus `language`/`cache_ttl`, derive `path` server-side + // (`inline_run_path`), and default the rest: a caller `hash`/ + // `lock`/`modules`/`path`/`dedicated_worker` would otherwise run + // or install unpinned code as the app identity. Reconstructing + // (vs nulling) keeps a new field defaulting safe. + RawCode { + content: raw_code.content, + language: raw_code.language, + path: Some(inline_run_path(path, &component)?), + cache_ttl: raw_code.cache_ttl, + ..Default::default() + } + }; (JobPayload::Code(raw_code), tag, None) } // inline script: run mode (deployed app) with an entry in `app_script`. @@ -4056,8 +4518,9 @@ async fn execute_component( // Apply runnable query parameters if provided if let Some(ref run_query) = payload.run_query_params { - if is_flow { - crate::jobs::process_flow_run_query_params(&mut tx, uuid, run_query).await?; + if let Some(flow_path) = flow_path.as_deref() { + crate::jobs::process_flow_run_query_params(&mut tx, uuid, &w_id, flow_path, run_query) + .await?; } } @@ -4216,8 +4679,11 @@ async fn upload_s3_file_from_app( request: axum::extract::Request, ) -> JsonResult { // Same path confinement as `execute_component`: without it a token scoped to app A - // could drive app B's upload policy. - if let Some(authed) = opt_authed.as_ref() { + // could drive app B's upload policy. A guest's waits for the app's mode, below. + if let Some(authed) = opt_authed + .as_ref() + .filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + { check_scopes(authed, || format!("apps:run:{}", path.to_path()))?; } let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex { @@ -4270,6 +4736,14 @@ async fn upload_s3_file_from_app( .map(|p| serde_json::from_value::(p).map_err(to_anyhow)) .transpose()? }; + let opt_authed = guest_caller_for_mode( + opt_authed, + policy + .as_ref() + .map(Policy::execution_mode) + .unwrap_or_default(), + path.to_path(), + )?; let user_db = UserDB::new(db.clone()); @@ -4281,7 +4755,7 @@ async fn upload_s3_file_from_app( let s3_inputs = policy.s3_inputs.as_ref().unwrap(); let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let on_behalf_authed = fetch_api_authed_from_permissioned_as( permissioned_as.clone(), @@ -4427,8 +4901,12 @@ async fn upload_s3_file_from_app( } } else { // backward compatibility (no policy) - // if no policy but logged in, use the user's auth to get the s3 resource - if let Some(authed) = opt_authed { + // if no policy but logged in, use the user's auth to get the s3 resource. A guest + // has no standing of its own to upload with, so without a policy it is refused + // exactly as an anonymous caller is. + if let Some(authed) = opt_authed + .filter(|a| !windmill_api_auth::scopes::has_guest_sentinel(a.scopes.as_deref())) + { let file_key = query .file_key .unwrap_or_else(|| get_random_file_name(query.file_extension)); @@ -4686,8 +5164,9 @@ async fn get_on_behalf_authed_from_app( }) }; + let opt_authed = guest_caller_for_mode(opt_authed.clone(), policy.execution_mode(), path)?; let (username, permissioned_as, email) = - get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed, &w_id, &db).await?; let on_behalf_authed = fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, Some(username)) @@ -4842,6 +5321,10 @@ fn check_app_s3_read_scope(opt_authed: &Option, path: &str) -> Result let Some(authed) = opt_authed.as_ref() else { return Ok(()); }; + // A guest's confinement waits for the app's mode (`get_on_behalf_authed_from_app`). + if windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) { + return Ok(()); + } check_scopes(authed, || format!("apps:run:{}", path)) .or_else(|_| check_scopes(authed, || format!("apps:read:{}", path))) } @@ -5194,7 +5677,45 @@ async fn app_load_csv_preview() -> Result<()> { )) } -fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { +/// The address to store beside the principal. Derived from it, never taken from the request, so +/// the stored copy can only ever agree with the principal — the drift it used to allow is what +/// this replaces. +/// +/// Written unconditionally, including for the versions that could derive it instead: a replica +/// predating that fallback fails outright when the key is absent, which would 400 every +/// anonymous, publisher and guest app for the length of a rolling deploy. The write is what +/// holds the key in place — see `docs/app-policy-email-removal.md`. +async fn stored_on_behalf_of_email(policy: &Policy, w_id: &str, db: &DB) -> Result> { + let Some(permissioned_as) = policy.on_behalf_of.as_deref() else { + return Ok(None); + }; + Ok(Some( + windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) + .await?, + )) +} + +/// The address to record in the `apps.on_behalf_of` audit entry: the one the app will run as, +/// when it is not the deployer's own. `None` when they match — a deployer handing an app their +/// own identity is not an on-behalf-of deploy. +/// +/// Reads the address `stored_on_behalf_of_email` just resolved rather than looking it up again, +/// so the audit row and the policy row can only ever name the same account. +fn audited_on_behalf_of(policy: &Policy, authed: &ApiAuthed) -> Option { + policy + .on_behalf_of_email + .as_deref() + .filter(|email| *email != authed.email) + .map(str::to_string) +} + +/// The identity an anonymous, publisher or guest execution runs as. +/// +/// `on_behalf_of_email` is optional: every write stores it, so it is present on anything this +/// release deployed, and it is only derived for a policy that predates that. Deriving is the +/// fallback rather than the rule so that removing the key later is a change of default, not a +/// change of behavior — see `docs/app-policy-email-removal.md`. +async fn get_on_behalf_of(policy: &Policy, w_id: &str, db: &DB) -> Result<(String, String)> { let permissioned_as = policy .on_behalf_of .as_ref() @@ -5205,16 +5726,15 @@ fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { ) })? .to_string(); - let email = policy - .on_behalf_of_email - .as_ref() - .ok_or_else(|| { - Error::BadRequest( - "on_behalf_of_email is missing in the app policy and is required for anonymous execution" - .to_string(), - ) - })? - .to_string(); + let email = match policy.on_behalf_of_email.as_deref() { + Some(email) => email.to_string(), + // Cached on purpose, up to one notify poll stale: the accepted dispatch case + // `get_email_from_permissioned_as` documents. + None => { + windmill_common::users::get_email_from_permissioned_as(&permissioned_as, w_id, db) + .await? + } + }; // Defence in depth against a policy that already carries a forged superadmin // sentinel (deployed before validation existed, or copied verbatim by a // workspace fork): the sentinels are internal-only and never a legitimate app @@ -5361,7 +5881,25 @@ async fn build_args( "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), "workspace" => Some(serde_json::to_value(&w_id)), "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - "author" => Some(serde_json::to_value(&policy.on_behalf_of_email)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } _ => { return Err(Error::BadRequest(format!( "context variable {} not allowed", @@ -5515,6 +6053,9 @@ mod embed_token_tests { "GET", ), ("/api/w/test/resources/list_search", "GET"), + // The metadata allowlist matches `resources/type/` by prefix, so a route + // added under it that is not a read must be denied by its method. + ("/api/w/test/resources/type/hub/pick/slack", "POST"), // Workspace-wide job enumeration/export must NOT be reachable — an app // reads only jobs it launched, by id (blocked via the app_embed sentinel). ("/api/w/test/jobs/list", "GET"), @@ -5608,6 +6149,10 @@ mod embed_token_tests { // the author declared it and the viewer consented. ("/api/w/test/resources/get_value/u/admin/r", "GET"), ("/api/w/test/variables/get_value/u/admin/v", "GET"), + // A chat UI's history for a chat-mode flow; RLS keeps it to the viewer's own. + ("/api/w/test/flow_conversations/list", "GET"), + ("/api/w/test/flow_conversations/some-uuid/messages", "GET"), + ("/api/w/test/flow_conversations/delete/some-uuid", "DELETE"), ]; for (path, method) in allowed { assert!( diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 2e2058ea80..6255b5eb13 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -567,6 +567,9 @@ async fn set_config( }; let mut tx = user_db.begin(&authed).await?; + if matches!(nc.trigger_kind, TriggerKind::Postgres) { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; + } sqlx::query!( r#" @@ -614,6 +617,9 @@ async fn ping_config( )>, ) -> Result<()> { let mut tx = user_db.begin(&authed).await?; + if matches!(trigger_kind, TriggerKind::Postgres) { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; + } sqlx::query!( r#" diff --git a/backend/windmill-api/src/csrf.rs b/backend/windmill-api/src/csrf.rs new file mode 100644 index 0000000000..4b2915f791 --- /dev/null +++ b/backend/windmill-api/src/csrf.rs @@ -0,0 +1,251 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::extract::FromRequestParts; +use axum::http::{header, request::Parts, Method}; +use axum::Extension; +use url::Url; +use windmill_common::error::Error; +use windmill_common::users::COOKIE_NAME; + +use crate::triggers::trigger_helpers::RunnableId; + +/// Whether a request is a cross-site GET authenticating on the session cookie alone. A GET +/// handler that runs a script by path resolves it through [`Self::script_runnable`], which +/// refuses a Hub script on such a request. +/// +/// The cookie is `SameSite=Lax`, so browsers attach it to cross-site top-level GET +/// navigations. A `hub/` path runs any public Hub script, and an argument written +/// `$var:` or `$res:` is resolved as the caller before the script sees it: such a +/// GET lets any page pick a generic Hub script and hand it the victim's secrets, which the job +/// can then send anywhere. +/// +/// Workspace scripts and flows are not refused, by choice, so that GET links to them keep +/// working. That is a scope decision, not a safety property: they still take attacker-chosen +/// arguments, `$var:` and `$res:` included, resolved as the victim. What bounds the exposure +/// is that the attacker needs a runnable path and can only run code the workspace deployed. +/// +/// The cookie is the only ambient credential. A bearer header is explicit, and so is the +/// `token` query parameter the webhook URLs carry — a cross-origin `EventSource` has no other +/// way to authenticate, since it cannot set headers. The checks run in `extract_token`'s +/// order, header before cookie, because that is the order it resolves them in: a request +/// carrying both a cookie and `token=` authenticates on the cookie and is therefore still +/// ambient, which is also why a valid `token=` link opened cross-site while signed in is +/// refused. +pub struct CrossSiteGetGuard(Option); + +impl CrossSiteGetGuard { + pub fn script_runnable(&self, script_path: &str) -> windmill_common::error::Result { + let runnable_id = RunnableId::from_script_path(script_path); + let (Some(signal), RunnableId::HubScript(_)) = (&self.0, &runnable_id) else { + return Ok(runnable_id); + }; + // The `Referer` leg is the one that can misfire, on a request that really was + // same-host: it compares against the hosts the backend can see, and a proxy that + // rewrites `Host` without setting `X-Forwarded-Host` leaves none of them matching + // what the browser addressed. Name the comparison so that shows up as a + // misconfiguration rather than as an unexplained 403. + if let CrossSite::RefererMismatch { referer, instance_hosts } = signal { + tracing::warn!( + referer_host = %referer, + ?instance_hosts, + "refusing a cross-site GET Hub script run inferred from Referer; if the request \ + was same-host, set `X-Forwarded-Host` on the proxy or configure `BASE_URL`" + ); + } + Err(Error::PermissionDenied( + "a cross-site GET request cannot run a Hub script with the session cookie, which takes \ + precedence over a `token` query parameter: pass the token in the `Authorization` \ + header, or open the link from the instance itself or from a browser with no Windmill \ + session" + .to_string(), + )) + } +} + +impl FromRequestParts for CrossSiteGetGuard { + type Rejection = std::convert::Infallible; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method != Method::GET { + return Ok(CrossSiteGetGuard(None)); + } + let Some(signal) = cross_site_signal(parts) else { + return Ok(CrossSiteGetGuard(None)); + }; + + let has_bearer = parts + .headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.starts_with("Bearer ")); + if has_bearer { + return Ok(CrossSiteGetGuard(None)); + } + + let has_session_cookie = + Extension::::from_request_parts(parts, state) + .await + .is_ok_and(|Extension(cookies)| cookies.get(COOKIE_NAME).is_some()); + Ok(CrossSiteGetGuard(has_session_cookie.then_some(signal))) + } +} + +enum CrossSite { + Declared, + RefererMismatch { referer: String, instance_hosts: Vec }, +} + +fn cross_site_signal(parts: &Parts) -> Option { + if let Some(site) = parts.headers.get("sec-fetch-site") { + return site + .as_bytes() + .eq_ignore_ascii_case(b"cross-site") + .then_some(CrossSite::Declared); + } + // Fetch Metadata rides only on potentially trustworthy URLs, so an instance served + // over plain http never receives `Sec-Fetch-Site` (nor does Safari before 16.4) while + // the cookie, not being `Secure` there either, still arrives. `Referer` is the only + // other thing a top-level GET navigation carries — `Origin` is not sent on one — so it + // is all that is left there, and it is weak: the default `strict-origin-when-cross- + // origin` policy already drops `Referer` on an https-to-http downgrade, so an https + // attacker page pointing a victim at a plain-http instance sends neither header. This + // leg catches an http-served attacker page and pre-16.4 Safari on https; the guard is + // load-bearing on https and best-effort at best on plain http. An absent `Referer` + // reads as not cross-site, matching how `Sec-Fetch-Site: none` (a bookmark, a typed + // URL) is treated. + let referer = referer_host(parts)?; + let instance_hosts: Vec = instance_hosts(parts).collect(); + (!instance_hosts + .iter() + .any(|host| host.eq_ignore_ascii_case(&referer))) + .then_some(CrossSite::RefererMismatch { referer, instance_hosts }) +} + +/// Every host a legitimate same-host request can name. `Host` alone is not enough: a +/// reverse proxy that forwards without preserving it (nginx `proxy_pass` with no +/// `proxy_set_header Host $host`) hands the backend the upstream's name, which no browser +/// `Referer` will ever match. None of these is browser-settable on a navigation — a +/// navigation carries no custom headers, and `BASE_URL` is instance config — so widening +/// the accepted set costs nothing. +fn instance_hosts(parts: &Parts) -> impl Iterator { + let base_url = windmill_common::BASE_URL.load(); + [ + request_host(parts), + header_host(parts, "x-forwarded-host"), + Url::parse(base_url.as_str()) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)), + ] + .into_iter() + .flatten() +} + +fn referer_host(parts: &Parts) -> Option { + let referer = parts.headers.get(header::REFERER)?.to_str().ok()?; + Url::parse(referer).ok()?.host_str().map(str::to_owned) +} + +fn request_host(parts: &Parts) -> Option { + if let Some(host) = parts.uri.host() { + return Some(host.to_owned()); + } + header_host(parts, header::HOST) +} + +fn header_host(parts: &Parts, name: impl header::AsHeaderName) -> Option { + host_of(parts.headers.get(name)?.to_str().ok()?) +} + +/// The host in a `Host`-shaped header value: `host[:port]`, where `host` may be a bracketed +/// IPv6 literal, and where a chain of proxies appends to `X-Forwarded-Host` so only the +/// first entry is the one the browser addressed. The port is split off by the URL parser +/// rather than by hand-rolling the bracket rules. +fn host_of(value: &str) -> Option { + let host = value.split(',').next()?.trim(); + Url::parse(&format!("http://{host}")) + .ok()? + .host_str() + .map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::{cross_site_signal, host_of, CrossSite}; + use axum::http::{request::Parts, Request}; + + fn parts(headers: &[(&str, &str)]) -> Parts { + let mut req = Request::get("/api/w/ws/jobs/run_wait_result/p/hub/1/x"); + for (name, value) in headers { + req = req.header(*name, *value); + } + req.body(()).unwrap().into_parts().0 + } + + #[test] + fn sec_fetch_site_decides_when_present() { + let declared = |site| cross_site_signal(&parts(&[("sec-fetch-site", site)])); + assert!(matches!(declared("cross-site"), Some(CrossSite::Declared))); + for site in ["same-origin", "same-site", "none"] { + assert!(declared(site).is_none(), "{site} is not cross-site"); + } + // The header outranks a `Referer` that disagrees with it. + let with_referer = parts(&[ + ("sec-fetch-site", "same-origin"), + ("host", "windmill.example"), + ("referer", "https://attacker.example/page"), + ]); + assert!(cross_site_signal(&with_referer).is_none()); + } + + #[test] + fn referer_stands_in_when_sec_fetch_site_is_absent() { + let signal = |headers: &[(&str, &str)]| cross_site_signal(&parts(headers)); + assert!(matches!( + signal(&[ + ("host", "windmill.example"), + ("referer", "https://attacker.example/p") + ]), + Some(CrossSite::RefererMismatch { .. }) + )); + // Ports differ between the frontend and the API, and do not make a request cross-site. + assert!(signal(&[ + ("host", "windmill.example:8000"), + ("referer", "http://windmill.example:3000/apps"), + ]) + .is_none()); + // A proxy that rewrote `Host` but forwarded the public name. + assert!(signal(&[ + ("host", "windmill-server.internal"), + ("x-forwarded-host", "windmill.example"), + ("referer", "https://windmill.example/apps"), + ]) + .is_none()); + assert!(signal(&[("host", "windmill.example")]).is_none()); + } + + #[test] + fn host_of_strips_port_brackets_and_proxy_chain() { + assert_eq!(host_of("windmill.example"), Some("windmill.example".into())); + assert_eq!( + host_of("windmill.example:8000"), + Some("windmill.example".into()) + ); + assert_eq!(host_of("[::1]:8000"), Some("[::1]".into())); + assert_eq!(host_of("[::1]"), Some("[::1]".into())); + assert_eq!( + host_of("windmill.example, proxy.internal"), + Some("windmill.example".into()) + ); + assert_eq!(host_of(""), None); + assert_eq!(host_of("not a host"), None); + } +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index d781e229d2..9619e96ebc 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -108,6 +108,9 @@ lazy_static::lazy_static! { (20260826214706, include_str!( "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260909163047, include_str!( + "../../migrations/20260909163047_workspace_delete_cascade_indexes.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } @@ -369,7 +372,10 @@ pub async fn migrate( } crate::live_migrations::custom_migrations(&mut custom_migrator).await?; - Ok(None) + Ok(Some(crate::live_migrations::spawn_background_migrations( + db.clone(), + killpill_rx, + ))) } pub async fn wait_for_migrations( diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index b44b0e64be..8bcbb56384 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -19,7 +19,7 @@ use windmill_common::{ error::{Error, Result}, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, - utils::strip_json_nul, + utils::{check_proper_path, strip_json_nul}, variables::{build_crypt, encrypt}, }; @@ -29,6 +29,7 @@ pub fn workspaced_service() -> Router { .route("/get/{kind}/{*path}", get(get_draft_for_user)) .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route("/move/{kind}/{*path}", post(move_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -214,6 +215,9 @@ fn list_drafts_query(all_users: bool) -> String { // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled (`password.username` is NULL). This // keeps the raw email out of the payload whenever a derived username exists. + // A null username means the legacy row downstream, so an owner that resolves to + // no name at all — an external JWT's subject has neither row — is dropped rather + // than surfaced as a second legacy entry. let draft_users = r#"CASE WHEN d.typ::text IN ('script', 'flow', 'app', 'raw_app') THEN ( SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END) NULLS LAST) @@ -221,6 +225,7 @@ fn list_drafts_query(all_users: bool) -> String { LEFT JOIN usr u ON u.workspace_id = du.workspace_id AND u.email = du.email LEFT JOIN password p ON p.email = du.email AND p.super_admin = true WHERE du.workspace_id = d.workspace_id AND du.path = d.path AND du.typ = d.typ + AND (du.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ) ELSE NULL END"#; // Default lists the user's own drafts AND the legacy NULL-email rows; with // `all_users` the filter is dropped to list every workspace draft. @@ -316,12 +321,71 @@ pub struct SaveDraftResponse { /// On `saved`: when the change was applied (client remembers it as the /// next `last_sync`). On `conflict`: the existing row's `created_at`. pub current_timestamp: chrono::DateTime, + /// `saved` only: where the write landed. Differs from the URL path when the item + /// had moved away from it; the editor follows it there. Absent when a delete found + /// nothing to remove and the caller cannot read the path it moved to. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, } -/// Apply the current user's draft at (workspace, kind, path): non-null `value` -/// upserts, `null` (or omitted) deletes. Either way, when the existing row is -/// newer than `last_sync` (and `force` is false) the op is skipped and the -/// response is `status = conflict` + the server's current timestamp. +/// The version a draft forked from, as the editors write it into `draft.value`. +/// Each kind names it differently and only one is ever set. +#[derive(Deserialize)] +struct DraftBaseVersion { + /// Scripts: hex-encoded script hash. + #[serde(default)] + parent_hash: Option, + /// Flows: `flow_version.id`. + #[serde(default)] + version_id: Option, + /// Apps / raw apps: `app_version.id`. + #[serde(default)] + parent_version: Option, +} + +impl DraftBaseVersion { + /// The base as the `draft.base` column stores it: one opaque text id whatever + /// the kind, so a reader compares it to the head without knowing the kind's + /// own field name or type. + fn as_text(&self, kind: UserDraftItemKind) -> Option { + use UserDraftItemKind::*; + match kind { + Script => self.parent_hash.clone(), + Flow => self.version_id.map(|v| v.to_string()), + _ => self.parent_version.map(|v| v.to_string()), + } + } +} + +/// The version this draft forked from, or `None` when it has none — a kind +/// that keeps no lineage, a malformed payload, or a draft that was never forked +/// from a deploy. Pure: no queries. +fn draft_lineage(kind: UserDraftItemKind, value: &str) -> Option { + use UserDraftItemKind::*; + if !matches!(kind, Script | Flow | App | RawApp) { + return None; + } + let base = serde_json::from_str::(value).ok()?; + let has_base = match kind { + Script => base + .parent_hash + .as_deref() + .and_then(|h| windmill_common::scripts::to_i64(h).ok()) + .is_some(), + Flow => base.version_id.is_some(), + _ => base.parent_version.is_some(), + }; + has_base.then_some(base) +} + +/// Apply the current user's draft: non-null `value` upserts, `null` (or +/// omitted) deletes. Either way, when the existing row is newer than +/// `last_sync` (and `force` is false) the op is skipped and the response is +/// `status = conflict` + the server's current timestamp. +/// +/// A save addressed to a path its item moved away from lands where the move took +/// the drafts (`draft_move`), unless the caller still has a draft of their own at +/// that path. The response names where it landed. async fn update_draft( authed: ApiAuthed, Extension(db): Extension, @@ -330,7 +394,7 @@ async fn update_draft( Json(req): Json, ) -> Result> { let email = &authed.email; - let path = path.to_path(); + let url_path = path.to_path(); // Saving a draft requires write permission on the underlying path. Deleting // (discarding) one's OWN draft does not: the email-scoped row belongs to the // authed user, so they can always discard it even after losing write access @@ -339,11 +403,68 @@ async fn update_draft( // touch the caller's own row. Legacy (NULL-email) rows aren't owned by anyone // — they keep the write gate. let is_own_discard = req.value.is_none() && !req.legacy; + // `legacy` targets the workspace-level row and is delete-only: an upsert writes the + // caller's own row whatever it says. Every read of that rule goes through this. + let legacy_delete = req.value.is_none() && req.legacy; + + // Whose row this write is for: the caller's, or the workspace-level one on a legacy + // DELETE (`legacy` is delete-only, so an upsert is the caller's own row either way). + // It picks both the record that applies — an item's move (`email IS NULL`) covers the + // legacy row too, since the same rename carried it — and the draft whose presence + // means this path is still the write's own. + let owner: Option<&str> = (!legacy_delete).then_some(email.as_str()); + // The caller's own draft-only move outranks the move of the deployed item. Only the + // kinds whose value carries a deploy target are ever recorded as moved, so for the + // rest this would be a guaranteed-empty query on the autosave hot path. + let moved_to = match kind.typed_path_field() { + None => None, + Some(_) => { + sqlx::query_scalar!( + r#"SELECT m.new_path FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ = $2 AND m.old_path = $3 + AND (m.email IS NULL OR m.email = $4) + AND NOT EXISTS ( + SELECT 1 FROM draft d + WHERE d.workspace_id = $1 AND d.typ = $2 AND d.path = $3 + AND d.email IS NOT DISTINCT FROM $4 + ) + ORDER BY m.email IS NULL + LIMIT 1"#, + &w_id, + kind as UserDraftItemKind, + url_path, + owner, + ) + .fetch_optional(&db) + .await? + } + }; + let path: &str = moved_to.as_deref().unwrap_or(url_path); + + // Everything past here writes, so the gate applies from here on. Answered + // without the path when the item moved: it may have gone somewhere the caller + // cannot see. if !is_own_discard { - require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + match require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await { + // Naming the move is for whoever was editing the item: it tells them why a + // save they were already making stopped landing. Someone who cannot read the + // path they addressed gets the plain denial, or the wording itself would + // answer whether an item was moved away from a path they only guessed at. + Err(Error::NotAuthorized(e)) if moved_to.is_some() => { + return Err(Error::NotAuthorized( + match require_can_read_path(&authed, &user_db, &w_id, kind, url_path).await { + Ok(()) => { + "this draft's item was moved to a path you cannot write".to_string() + } + Err(_) => e, + }, + )); + } + other => other?, + } } - let applied_at = if let Some(value) = &req.value { + let applied = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext // (see `encrypt_secret_variable_value`). let serialized = if kind == UserDraftItemKind::Variable { @@ -355,19 +476,49 @@ async fn update_draft( // escape and later make any `->>`/`to_jsonb` extraction raise `22P05`. // Strip it here so a NUL never reaches the column. let serialized = strip_json_nul(&serialized); + // `base` is derived here from the value's per-kind field rather than sent + // by the client, so every writer (editors, chat, CLI) fills it the same way. + // Read from the sanitized text, which is what the value column gets: a NUL in the + // lineage field otherwise costs the draft its base (the hash no longer parses), + // leaving a draft that reads as up to date whatever the head is. + let base = draft_lineage(kind, serialized.as_ref()).and_then(|l| l.as_text(kind)); // Upsert. The conflict check rides on the DO UPDATE WHERE clause — // when the row is newer than `last_sync`, RETURNING yields nothing. // `created_at` defaults to `now()` but the migration overrides it ($8) // so a migrated draft keeps its original age instead of jumping to top. - sqlx::query_scalar!( - r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - VALUES ($1, $2, $3, $4, $5::text::json, COALESCE($8::timestamptz, now())) + // + // A moved save ($10) carries the path keys its editor had before the move. + // One naming the path it addressed ($11) follows to where it landed; a draft + // already there keeps the keys the move gave it. A pre-sanitizer NUL escape + // in that draft makes `to_jsonb` raise, so it takes the incoming keys. + sqlx::query!( + r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base) + VALUES ($1, $2, $3::text, $4, + CASE WHEN $10::bool + THEN to_json($5::text::jsonb || jsonb_strip_nulls(jsonb_build_object( + 'path', CASE WHEN $5::text::jsonb -> 'path' = to_jsonb($11::text) + THEN to_jsonb($3::text) END, + 'draft_path', CASE WHEN $5::text::jsonb -> 'draft_path' = to_jsonb($11::text) + THEN to_jsonb($3::text) END))) + ELSE $5::text::json + END, + COALESCE($8::timestamptz, now()), $9) ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at + DO UPDATE SET value = CASE + WHEN NOT $10::bool + OR position(chr(92) || 'u0000' in replace(draft.value::text, chr(92) || chr(92), '')) > 0 + THEN EXCLUDED.value + ELSE to_json((to_jsonb(EXCLUDED.value) - 'path' - 'draft_path') + || jsonb_strip_nulls(jsonb_build_object( + 'path', to_jsonb(draft.value) -> 'path', + 'draft_path', to_jsonb(draft.value) -> 'draft_path'))) + END, + created_at = EXCLUDED.created_at, + base = EXCLUDED.base WHERE $7::bool = true OR $6::timestamptz IS NULL OR draft.created_at <= $6::timestamptz - RETURNING created_at"#, + RETURNING path, created_at"#, &w_id, email, path, @@ -376,9 +527,13 @@ async fn update_draft( req.last_sync, req.force, req.created_at, + base.as_deref(), + moved_to.is_some(), + url_path, ) .fetch_optional(&db) .await? + .map(|r| (r.created_at, Some(r.path))) } else { // Delete, same conflict rule in the WHERE clause. Returns NULL when // the row was too new (conflict) OR already absent (idempotent) — @@ -399,16 +554,20 @@ async fn update_draft( kind as UserDraftItemKind, req.last_sync, req.force, - req.legacy, + legacy_delete, ) .fetch_optional(&db) .await? + // Named for the same reason an upsert is: the editor that discarded is still on + // the path the item left, and reloading there would land on nothing. + .map(|ts| (ts, moved_to.clone())) }; - if let Some(ts) = applied_at { + if let Some((ts, path)) = applied { return Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Saved, current_timestamp: ts, + path, })); } @@ -424,7 +583,7 @@ async fn update_draft( email, path, kind as UserDraftItemKind, - req.legacy, + legacy_delete, ) .fetch_optional(&db) .await?; @@ -433,20 +592,307 @@ async fn update_draft( Some(ts) => Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Conflict, current_timestamp: ts, + path: None, })), // Delete + nothing-was-there ⇒ report success with server's NOW(). None => { let now = sqlx::query_scalar!(r#"SELECT now() as "now!""#) .fetch_one(&db) .await?; + // A retry of a routed discard whose answer was lost lands here: the row is + // gone but the editor is still on the path the item left, so it needs the + // destination as much as the first attempt did. Unlike the arm above there is + // no deleted row proving the caller ever held that draft, and an own discard + // is not gated, so this names a path to someone who may have none of it: the + // read gate, which is what keeps a path from being disclosed elsewhere in this + // module. Without it, discarding at a guessed path reads `draft_move`. + let disclosed = match moved_to { + Some(dest) => { + match require_can_read_path(&authed, &user_db, &w_id, kind, &dest).await { + Ok(()) => Some(dest), + Err(Error::NotFound(_)) + | Err(Error::NotAuthorized(_)) + | Err(Error::BadRequest(_)) => None, + Err(e) => return Err(e), + } + } + None => None, + }; Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Saved, current_timestamp: now, + path: disclosed, })) } } } +#[derive(Deserialize)] +pub struct MoveDraftRequest { + pub new_path: String, + /// Also restate the draft's summary, so the same drawer that renames a + /// deployed item can retitle a draft-only one. + #[serde(default)] + pub summary: Option, +} + +/// Relocate the authed user's own DRAFT-ONLY item. Such an item is nothing but +/// its draft row, so moving it is a rewrite of that row's path plus both path +/// keys inside its value — there is no deployed row, schedule or trigger to +/// cascade to. +/// +/// The owner's own open editor follows: its next save, still addressed to the old +/// path, lands at the new one through the move record, and it is told where. +/// +/// Scoped to the caller's own row on purpose: two users can each have a draft +/// at the same never-deployed path, and those are two separate items. +/// +/// A DEPLOYED item must move through its own deploy endpoint instead, which +/// cascades everything that references the path and carries every draft along. +async fn move_draft( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Json(req): Json, +) -> Result { + let path = path.to_path(); + let new_path = req.new_path.as_str(); + // Only the full-page editor kinds, which is exactly the set that has a typed + // path to rewrite. Reading the movable set off the same mapping the rewrite + // uses keeps them from drifting apart: a resource, a variable or a trigger + // keeps its deploy target in `value.path` with no editor to stage a rename, + // so moving one would leave the real target naming the old location and the + // next deploy would recreate it where it came from. + let (Some(typed_field), Some(mirror_field)) = + (kind.typed_path_field(), kind.mirror_path_field()) + else { + return Err(Error::BadRequest(format!( + "moving a {kind:?} draft is not supported — only scripts, flows and apps" + ))); + }; + // Validate before authorizing: `require_can_write_path` is not a format check + // (an admin returns immediately, and a user returns early inside their own + // namespace), so without this a malformed path is stored as-is, and an over-long + // or NUL-bearing one reaches Postgres as a raw server error. The source is a URL + // segment and arrives decoded, so it needs the same check as the destination. + check_proper_path(path)?; + check_proper_path(new_path)?; + // A summary-only edit is a legitimate use of this endpoint: the drawer edits + // both fields, and for a draft-only script the path it posts back is the row + // path unchanged (`list_scripts` only reports `draft_path` when it differs). + // Returning early on the path alone would drop the new summary silently. + if new_path == path && req.summary.is_none() { + return Ok("unchanged".to_string()); + } + require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + if new_path != path { + require_can_write_path(&authed, &db, &user_db, &w_id, kind, new_path).await?; + } + + if let Some(table) = kind.deployed_table() { + // `table` is from the closed `deployed_table()` enum, never user input. + // Archived and soft-deleted rows keep sitting at their path — a script + // move archives its parent in place — so an existence check that counted + // them would refuse a move away from, or into, a path nothing occupies. + // `create_script_internal` resolves its own path clashes the same way. + let archived_filter = if table == "script" { + " AND NOT archived AND NOT deleted" + } else { + "" + }; + let query = format!( + "SELECT 1 FROM {table} WHERE path = $1 AND workspace_id = $2{archived_filter} LIMIT 1" + ); + let mut tx = user_db.clone().begin(&authed).await?; + let deployed_at_old = sqlx::query_scalar::<_, i32>(&query) + .bind(path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + let deployed_at_new = sqlx::query_scalar::<_, i32>(&query) + .bind(new_path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if deployed_at_old.is_some() { + return Err(Error::BadRequest(format!( + "'{path}' is deployed — move it from its editor so schedules and triggers follow" + ))); + } + if deployed_at_new.is_some() { + return Err(Error::BadRequest(format!( + "'{new_path}' already has a deployed item — moving there would turn this into a draft on top of it" + ))); + } + } + + // A NUL in the summary reaches Postgres as a raw byte and fails the statement with + // an encoding error, so it is dropped here the way `strip_json_nul` drops one from a + // draft value. `json` cannot store it either. + let summary = req.summary.as_ref().map(|s| s.replace('\0', "")); + + // A classic app and a raw app share the `app` table, so a draft of either kind + // occupies the destination for both: deploying there deletes the caller's drafts + // of both kinds, taking the item that lost the collision with it. + let collision_typs: Vec<&str> = match kind { + UserDraftItemKind::App | UserDraftItemKind::RawApp => vec![ + UserDraftItemKind::App.as_str(), + UserDraftItemKind::RawApp.as_str(), + ], + _ => vec![kind.as_str()], + }; + + // One transaction with the move record, so a save addressed to the old path + // never sees the row gone without knowing where it went. + let mut tx = db.begin().await?; + let moved = sqlx::query_scalar!( + r#"UPDATE draft + SET path = $3, + -- Both path keys, not just the typed one: the editors mirror the + -- typed path into the other while it differs from the row's path, + -- and the loaders prefer the mirror — left naming the old location + -- it un-does this move on the next save. `create_missing = false` + -- on both, so a draft carrying only one keeps only one. + value = to_json( + jsonb_set( + jsonb_set( + CASE WHEN $7::text IS NULL THEN to_jsonb(value) + ELSE jsonb_set(to_jsonb(value), ARRAY['summary'], to_jsonb($7::text)) + END, + ARRAY[$5::text], to_jsonb($3::text), false + ), + ARRAY[$8::text], to_jsonb($3::text), false + ) + ) + WHERE workspace_id = $1 + AND path = $2 + AND typ = $4 + AND email = $6 + -- A pre-sanitizer NUL escape makes `to_jsonb` raise 22P05. Excluded + -- here so the statement can't 500; reported below instead. Unlike the + -- passive carry, rewriting the value IS this operation, so skipping it + -- silently would move the row and leave its typed path stale. + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0 + -- Skipped on a summary-only edit, where the "target" row is this + -- row and the guard would refuse the update against itself. + AND ($2 = $3 OR NOT EXISTS ( + SELECT 1 FROM draft o + WHERE o.workspace_id = $1 AND o.path = $3 AND o.typ::text = ANY($9::text[]) + -- Of this kind only the caller's own row and the legacy one collide: + -- teammates' drafts of one item share its path by design, but a deploy + -- there wipes those two together, so a second would discard edits the + -- caller never saw. The other app kind is a different item on the same + -- deployed path, so it collides whoever owns it. + AND (o.typ <> $4 OR o.email = $6 OR o.email IS NULL) + )) + RETURNING id"#, + &w_id, + path, + new_path, + kind as UserDraftItemKind, + typed_field, + &authed.email, + summary, + mirror_field, + &collision_typs as &[&str], + ) + .fetch_optional(&mut *tx) + .await?; + if moved.is_some() && new_path != path { + windmill_common::user_drafts::record_draft_move( + &mut tx, + &w_id, + &[kind], + path, + new_path, + Some(&authed.email), + ) + .await?; + } + tx.commit().await?; + + if moved.is_none() { + let row = sqlx::query!( + r#"SELECT + -- The guard's own predicate, ordered own row, then legacy, then another + -- user's other-kind row: each needs a different remedy, and a bare + -- LIMIT 1 would name an arbitrary one. + (SELECT typ::text FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) + AND (typ <> $2 OR email = $4 OR email IS NULL) + ORDER BY CASE WHEN email = $4 THEN 0 WHEN email IS NULL THEN 1 ELSE 2 END + LIMIT 1) as "at_target", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) AND email = $4) as "at_target_own!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) AND email IS NULL) as "at_target_legacy!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5 + AND typ = $2 AND email = $4 + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) > 0 + ) as "poisoned!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5 + AND typ = $2 AND email IS NULL) as "legacy!" "#, + &w_id, + kind as UserDraftItemKind, + new_path, + &authed.email, + path, + &collision_typs as &[&str], + ) + .fetch_one(&db) + .await?; + return Err(Error::BadRequest(if row.poisoned { + // This endpoint also serves a summary-only edit, so name the operation + // the caller actually asked for rather than always saying "moved". + let attempted = if new_path == path { "updated" } else { "moved" }; + format!( + "'{path}' contains a NUL character and predates the sanitizer, so it cannot be \ + {attempted}. Reopen it, re-save to rewrite it cleanly, then retry." + ) + } else if row.legacy { + // The home list synthesizes a draft-only row for the legacy draft with the + // caller's own name on it, so this is reachable from the row menu. Only an + // admin can claim or discard that row, and only from the drafts page. + format!( + "'{path}' is a legacy workspace draft with no owner, so it cannot be moved. \ + A workspace admin can claim or discard it on the Review & deploy drafts page." + ) + } else if let Some(occupant) = row.at_target.filter(|_| new_path != path) { + // Naming the kind matters for the app pair: a classic-app draft refusing a + // raw-app move is invisible in the raw-app list the caller is looking at. + let occupant = occupant.replace('_', " "); + if row.at_target_own { + format!("You already have a draft at '{new_path}' ({occupant})") + } else if row.at_target_legacy { + // An ownerless row the caller cannot clear themselves, so send them to + // the one place it can be resolved rather than to "discard your draft". + format!( + "A legacy workspace draft with no owner is already at '{new_path}' \ + ({occupant}). A workspace admin can claim or discard it on the Review & \ + deploy drafts page." + ) + } else { + // The other app kind, owned by someone else: one deployed path cannot hold + // both, so this is the other item's path, not a teammate's copy of this one. + format!( + "'{new_path}' holds another user's {occupant} draft, and an app and a raw \ + app cannot share a path. Pick another path, or ask them to move or \ + discard theirs." + ) + } + } else { + format!("You have no draft at '{path}'") + })); + } + + if new_path == path { + return Ok(format!("updated draft {path}")); + } + Ok(format!("moved draft {path} to {new_path}")) +} + #[derive(Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub enum MigrateLegacyDraftAction { @@ -500,12 +946,12 @@ async fn migrate_legacy_draft( r#"WITH legacy AS ( DELETE FROM draft WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL - RETURNING value + RETURNING value, base ) - INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - SELECT $1, $4, $2, $3, value, now() FROM legacy + INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base) + SELECT $1, $4, $2, $3, value, now(), base FROM legacy ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = now() + DO UPDATE SET value = EXCLUDED.value, created_at = now(), base = EXCLUDED.base RETURNING 1 as "one!""#, &w_id, path, @@ -763,8 +1209,8 @@ async fn require_can_write_path( /// Resolves to `Ok(())` if `authed` can read at `path`. Three layers: /// 1. admin → always. -/// 2. Path-prefix match against own `u/{username}` or any folder in -/// `authed.folders` (the precomputed read set, with groups + direct +/// 2. Path-prefix match against own `u/{username}`, a group in `authed.groups`, or any +/// folder in `authed.folders` (the precomputed read set, with groups + direct /// grants already factored in). /// 3. RLS-aware `SELECT 1` against the backing table — covers item-level /// extra_perms grants that bypass folder/owner checks. @@ -789,6 +1235,10 @@ async fn require_can_read_path( if parts.len() >= 2 { match parts[0] { "u" if parts[1] == authed.username => return Ok(()), + // As `require_can_write_path` and the `see_member` RLS policy read it: a + // draft-only `g/` path has no row for the probe below to fall back on, so + // without this a member cannot see a draft their group owns. + "g" if authed.groups.iter().any(|g| g == parts[1]) => return Ok(()), "f" => { let folder = parts[1]; if authed.folders.iter().any(|(name, _, _)| name == folder) { diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs index 0451d88699..d1653d9aa9 100644 --- a/backend/windmill-api/src/git_sync_oss.rs +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -10,6 +10,11 @@ pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] +pub fn workspaced_git_sync_service() -> Router { + Router::new() +} + #[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index ccb530ce20..633ad88a34 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -6,13 +6,14 @@ use axum::{ http::{request::Parts, StatusCode}, response::{IntoResponse, Response}, routing::{get, post}, - Router, + Extension, Router, }; use serde::{Deserialize, Deserializer, Serialize}; use windmill_common::{ error::{to_anyhow, Error}, + global_settings::{load_value_from_global_settings, DISABLE_HUB_SETTING}, utils::require_admin, - HUB_BASE_URL, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, }; pub fn workspaced_service() -> Router { @@ -48,6 +49,7 @@ pub fn workspaced_service() -> Router { post(discard_project_update), ) .route("/project", get(get_project_by_source)) + .route("/projects", get(list_projects)) } #[derive(Deserialize)] @@ -548,6 +550,84 @@ async fn get_project_by_source(ctx: HubPublishCtx) -> Result bool { + fn host_of(url: &str) -> Option { + let parsed = url::Url::parse(url.trim()).ok()?; + if !matches!(parsed.scheme(), "http" | "https") { + return None; + } + Some( + parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(), + ) + } + match (host_of(hub), host_of(DEFAULT_HUB_BASE_URL)) { + (Some(host), Some(default_host)) => host == default_host, + _ => false, + } +} + +// The hub's project catalogue. Read by any workspace member rather than through +// `HubPublishCtx`, which requires an admin: nothing here is workspace-scoped or +// publishing-related. It exists at all because the hub's listing endpoint sends no +// CORS header, so the browser cannot read it directly the way it reads a single +// project. `accept: application/json` is what makes the hub answer with JSON. +// +// The caller's token is sent only to a hub this instance was pointed at deliberately. +// Every other route here is admin-only; this one is not, so forwarding a member's +// bearer token to `hub.windmill.dev` would put a credential replayable against this +// instance on a host outside it — for a listing that needs no credential at all. +async fn list_projects( + _authed: ApiAuthed, + Extension(db): Extension, + Tokened { token }: Tokened, +) -> Result { + // `disable_hub` turns the hub off for a closed instance, and this handler makes an + // outbound request. The frontend hides its entry points on the same setting, but that + // is presentation: an authenticated member can call this route directly, so the refusal + // has to live here. + let disabled = load_value_from_global_settings(&db, DISABLE_HUB_SETTING) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if disabled { + return Err(Error::BadRequest( + "The hub is disabled on this instance".to_string(), + )); + } + + let hub = (**HUB_BASE_URL.load()).clone(); + let url = format!("{}/projects", hub); + let mut req = HTTP_CLIENT.get(&url).header("accept", "application/json"); + if !is_public_hub(&hub) { + req = req.bearer_auth(&token); + } + let res = req + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + async fn submit_project( ctx: HubPublishCtx, Path((_workspace, slug)): Path<(String, ProjectSlug)>, @@ -645,3 +725,42 @@ async fn forward_to_hub( Ok((status, text)) } + +#[cfg(test)] +mod tests { + use super::is_public_hub; + + #[test] + fn public_hub_recognized_in_every_spelling() { + // The predicate decides whether a workspace member's bearer token leaves the + // instance, so both directions matter: a miss on the public hub sends the token + // to windmill.dev, and a false match withholds it from a private hub that needs it. + // Every spelling here is one `hub_base_url` can hold and `reqwest` will still send. + for hub in [ + "https://hub.windmill.dev", + "http://hub.windmill.dev/", + "HTTPS://hub.windmill.dev", + "https://HUB.WINDMILL.DEV", + "https://hub.windmill.dev:443", + "https://hub.windmill.dev.", + "https://hub.windmill.dev/some/path", + " https://hub.windmill.dev ", + ] { + assert!(is_public_hub(hub), "{hub} should be the public hub"); + } + for hub in [ + "https://hub.internal.example", + "https://hub.windmill.dev.evil.example", + "https://windmill.dev", + // The host is what the request goes to, whatever precedes the `@`. + "https://hub.windmill.dev@hub.internal.example", + // Unparseable, or not a scheme a request can be built from. Grouped with the + // private hubs because the caller then attaches the token, which is harmless here: + // `reqwest` rejects the same value before opening a connection. + "hub.windmill.dev", + "ftp://hub.windmill.dev", + ] { + assert!(!is_public_hub(hub), "{hub} should not be the public hub"); + } + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a30cffbfbf..18ddc0b787 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -18,6 +18,7 @@ use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; use sha2::{Digest, Sha256}; +use std::borrow::Cow; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; @@ -72,6 +73,7 @@ use crate::{ args::{self, RawWebhookArgs}, auth::{OptTokened, Tokened}, concurrency_groups::join_concurrency_key, + csrf::CrossSiteGetGuard, db::{ApiAuthed, DB}, triggers::trigger_helpers::RunnableId, users::{ @@ -105,9 +107,13 @@ use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, flow_status::{Approval, ApprovalConditions, FlowStatus, FlowStatusModule}, - flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, + flows::{ + add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue, + Suspend, + }, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, + query_builders, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, @@ -137,6 +143,7 @@ pub fn workspaced_service() -> Router { .route("/run_progress/{id}", get(get_run_progress)) .route("/run_assets/{id}", get(list_run_assets)) .route("/dbt_graph/{id}", get(get_dbt_run_graph)) + .route("/dbt_column_lineage/{id}", get(get_dbt_run_column_lineage)) .route("/dbt_resumable/{id}", get(get_dbt_resumable)) .route( "/dbt_resumable_script/p/{*script_path}", @@ -889,21 +896,27 @@ struct AssetProgress { error: Option, } -/// The asset graph as one run saw it. Pinning to a job needs the full job-read -/// contract, so it lives on `require_job_read_access` here rather than as a -/// parameter on `/assets/graph`. See docs/dbt-runtime.md. -async fn get_dbt_run_graph( - authed: ApiAuthed, - OptViewToken(view_token): OptViewToken, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Query(q): Query, -) -> error::JsonResult { +/// Which project version a dbt view pins to for this job, once the caller has +/// been shown to be entitled to it. +/// +/// `Ok(None)` is "answer unpinned", not a refusal: a job that stored no graph of +/// its own — and one that has aged out of retention — is served the deployed +/// version rather than an error, so a run page keeps drawing after the run is +/// gone. Pinning needs the full job-read contract, which is why it lives on +/// `require_job_read_access` here rather than as a parameter on `/assets/*`. +/// See docs/dbt-runtime.md. +async fn dbt_pinned_run( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + job_id: Uuid, + view_token: Option<&str>, +) -> error::Result> { // The scope domain comes from the URL segment, so `/jobs` asks a scoped token // for `jobs:read` alone while the body returned is asset data. Both are // required: the job gate below reaches this run, this reaches assets at all. - check_scopes(&authed, || "assets:read".to_string())?; + check_scopes(authed, || "assets:read".to_string())?; let job = sqlx::query!( r#"SELECT created_by, runnable_path, CASE WHEN kind = 'script' THEN runnable_id END AS script_hash, @@ -916,42 +929,70 @@ async fn get_dbt_run_graph( AND g.script_hash IS NULL) AS "editor_graph!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, job_id, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; - // No such job: answer the unpinned graph rather than 404, so a run page whose - // job has aged out of retention still draws the deployed version instead of - // an error. Reachable only with `assets:read`, which is exactly what - // `/assets/graph` would have cost for the same answer. + // Unpinned rather than 404 for a job that is gone. Reachable only with + // `assets:read`, which is exactly what the unpinned route would have cost + // for the same answer. let Some(job) = job else { - return windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, None).await; + return Ok(None); }; require_job_read_access( - &db, - &user_db, - &authed, - &w_id, + db, + user_db, + authed, + w_id, &job_id, &job.created_by, - view_token.as_deref(), + view_token, ) .await?; // A preview or flow job names no deployed version, so there is usually no // graph to pin to and the workspace one answers. The exception is a job that // parsed one itself, which is what the dbt editor's refresh is: its graph // belongs to that job alone and nothing else can reach it. - let pinned = job + Ok(job .runnable_path .filter(|_| job.script_hash.is_some() || job.editor_graph) .map(|path| windmill_api_assets::PinnedRun { job_id, script_path: path, script_hash: job.script_hash, - }); + })) +} + +/// The asset graph as one run saw it. +async fn get_dbt_run_graph( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(q): Query, +) -> error::JsonResult { + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, pinned).await } +/// The column lineage a set of relations sits in as one run saw it — the same +/// pin as `get_dbt_run_graph`, for the trace drawn beside a node of that graph. +async fn get_dbt_run_column_lineage( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(pairs): Query>, +) -> error::JsonResult { + let q = windmill_api_assets::ColumnLineageQuery::from_query_pairs(pairs)?; + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; + windmill_api_assets::dbt_column_lineage_for(&authed, &w_id, user_db, q, pinned).await +} + /// Whether a `dbt retry` submitted by this caller would resume THIS run. /// /// One failure is saved per script per execution principal, so a page showing an @@ -1592,7 +1633,11 @@ pub(crate) async fn require_job_read_access( // this token, and letting it reach any job merely visible to the viewer would // expose unrelated runs' results/logs. Stop at the launched-by-viewer grant. // NotFound (not PermissionDenied) so the untrusted app can't probe job existence. - if windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) { + // A guest stops here too: it has no membership behind it, so a share token whose + // audience is the workspace's members must not read for it either. + if windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) + || windmill_api_auth::scopes::has_guest_sentinel(authed.scopes.as_deref()) + { return Err(Error::NotFound(format!("Job {job_id} not found"))); } @@ -4324,12 +4369,12 @@ async fn count_completed_jobs_detail( Query(query): Query, ) -> error::JsonResult { let mut sqlb = SqlBuilder::select_from("v2_job_completed"); - //FOR RLS - sqlb.join("v2_job USING (id)"); sqlb.field("COUNT(*) as count"); + // Filtering on v2_job.workspace_id instead would keep the planner off + // ix_job_workspace_id_completed_at_all and scan the whole retention window. if !(w_id == "admins" && query.all_workspaces.unwrap_or(false)) { - sqlb.and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); + sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)); } if let Some(after_s_ago) = query.completed_after_s_ago { @@ -4348,6 +4393,7 @@ async fn count_completed_jobs_detail( } if let Some(tags) = query.tags { + sqlb.join("v2_job USING (id)"); sqlb.and_where_in( "v2_job.tag", &tags.split(",").map(|t| quote(t)).collect::>(), @@ -4355,7 +4401,19 @@ async fn count_completed_jobs_detail( } let sql = sqlb.sql()?; - let stats = sqlx::query_scalar::<_, i64>(&sql).fetch_one(&db).await?; + let mut tx = db.begin().await?; + set_list_jobs_statement_timeout(&mut tx).await?; + let stats = sqlx::query_scalar::<_, i64>(&sql) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + list_jobs_timeout_error( + e, + "Counting completed jobs", + "Lower completed_after_s_ago or narrow the filters.", + ) + })?; + tx.commit().await?; Ok(Json(stats)) } @@ -4375,6 +4433,42 @@ async fn count_completed_jobs( )) } +lazy_static::lazy_static! { + /// 0 keeps the connection-wide statement_timeout. + static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 = + std::env::var("LIST_JOBS_STATEMENT_TIMEOUT_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(30); +} + +/// A client that gives up does not cancel its query, so without this bound every retry of a +/// slow filter stacks another scan running until the connection-wide 5min timeout. +async fn set_list_jobs_statement_timeout(tx: &mut Transaction<'_, Postgres>) -> error::Result<()> { + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + if timeout_secs > 0 { + sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'")) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +fn list_jobs_timeout_error(e: sqlx::Error, action: &str, hint: &str) -> Error { + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + match e { + sqlx::Error::Database(ref db_err) + if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") => + { + Error::Generic( + StatusCode::BAD_REQUEST, + format!("{action} took more than {timeout_secs}s and was stopped. {hint}"), + ) + } + e => e.into(), + } +} + async fn list_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4491,11 +4585,15 @@ async fn list_jobs( }; // tracing::info!("sql: {}", &sql); let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + set_list_jobs_statement_timeout(&mut tx).await?; let jobs: Vec = sqlx::query_as(&sql) .fetch_all(&mut *tx) .warn_after_seconds_with_sql(5, format!("list_jobs: {}", sql)) - .await?; + .await + .map_err(|e| { + list_jobs_timeout_error(e, "Listing jobs", "Set a start date or narrow the filters.") + })?; tx.commit().await?; Ok(Json(jobs.into_iter().map(From::from).collect())) @@ -4785,6 +4883,11 @@ struct ApprovalInfo { user_auth_required: bool, #[serde(skip_serializing_if = "Option::is_none")] hide_cancel: Option, + skin: ApprovalSkin, + #[serde(skip_serializing_if = "Option::is_none")] + step_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_summary: Option, approvers: Vec, /// Share-read-link token for the flow, minted only for callers allowed to view this /// approval. Lets an authenticated workspace-member approver open the run details of @@ -4838,6 +4941,48 @@ fn can_approve_step( } } +/// The latest approval step the run has passed: a step before the current `step` that ran +/// rather than being skipped. Steps from `step` on don't count, because while an approval is +/// pending the step after it already holds the `WaitingForEvents` status. +fn last_reached_approval_step<'a>( + flow: &'a FlowValue, + status: &FlowStatus, +) -> Option<&'a FlowModule> { + flow.modules + .iter() + .zip(status.modules.iter()) + .take(usize::try_from(status.step).unwrap_or(0)) + .rev() + .filter(|(_, m)| matches!(m, FlowStatusModule::Success { skipped: false, .. })) + .map(|(module, _)| module) + .find(|module| module.suspend.is_some()) +} + +/// The approval conditions a step's own settings give, as the worker records them when the step +/// suspends. The worker drops them from the run once the step is approved, so a run that has +/// moved on is gated by these. Groups computed by an expression can't be re-evaluated outside +/// the run, so such a step falls back to any signed-in user. +fn approval_conditions_from_settings(suspend: &Suspend) -> Option { + let user_auth_required = suspend.user_auth_required.unwrap_or(false); + let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + if !user_auth_required && !self_approval_disabled { + return None; + } + let user_groups_required = match &suspend.user_groups_required { + Some(InputTransform::Static { value }) if user_auth_required => { + serde_json::from_str(value.get()).unwrap_or_default() + } + _ => vec![], + }; + Some(ApprovalConditions { user_auth_required, user_groups_required, self_approval_disabled }) +} + +/// How the approval step presents itself on the approval page. +struct ApprovalStepView { + skin: ApprovalSkin, + summary: Option, +} + async fn get_approval_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -4866,13 +5011,33 @@ async fn get_approval_info( script_path: Option, email: String, flow_status: Option, - workflow_as_code_status: Option, + // `v2_job_status` only holds a run that hasn't finished, so the fields below also read + // the completed run's status: a finished run's page keeps its skin and, for workflows as + // code, its description, still gated by the approval conditions the run had. + completed_flow_status: Option, + is_wac: bool, + wac_approval: Option, + approval_conditions: Option, + flow_summary: Option, } let row = sqlx::query_as::<_, ApprovalJobRow>( "SELECT j.id, j.runnable_path as script_path, j.permissioned_as_email as email, - s.flow_status, s.workflow_as_code_status + s.flow_status, + c.flow_status AS completed_flow_status, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status) IS NOT NULL + AS is_wac, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status)->'_approval' + AS wac_approval, + COALESCE(s.flow_status, c.flow_status)->'approval_conditions' + AS approval_conditions, + NULLIF(COALESCE(f.summary, sc.summary), '') AS flow_summary FROM v2_job j LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN flow f + ON j.kind = 'flow' AND f.workspace_id = j.workspace_id AND f.path = j.runnable_path + LEFT JOIN script sc + ON j.kind = 'script' AND sc.workspace_id = j.workspace_id AND sc.hash = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", ) .bind(&job_id) @@ -4881,31 +5046,31 @@ async fn get_approval_info( .await? .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; - let is_wac = row.workflow_as_code_status.is_some(); + let is_wac = row.is_wac; + let run_ac = row + .approval_conditions + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); // Extract approval info based on WAC vs classic flow - let (form_schema, description, default_args, enums, approval_conditions, hide_cancel) = + let (form_schema, description, default_args, enums, approval_conditions, hide_cancel, step) = if is_wac { - let approval_meta = row - .workflow_as_code_status - .as_ref() - .and_then(|v| v.get("_approval")); + let approval_meta = row.wac_approval.as_ref(); let form = approval_meta.and_then(|m| m.get("form").cloned()); let default_args = approval_meta.and_then(|m| m.get("default_args").cloned()); let enums = approval_meta.and_then(|m| m.get("enums").cloned()); let description = approval_meta.and_then(|m| m.get("description").cloned()); - let ac = row - .flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - (form, description, default_args, enums, ac, None) + let skin = approval_meta + .and_then(|m| m.get("skin")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .unwrap_or_default(); + let step = Some(ApprovalStepView { skin, summary: None }); + (form, description, default_args, enums, run_ac, None, step) } else { let fs = row .flow_status .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()); - let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); // For classic flows, form/description come from the flow definition and step result let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); @@ -4965,6 +5130,28 @@ async fn get_approval_info( .and_then(|s| s.resume_form.as_ref()) .map(|rf| serde_json::json!(rf)); let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + let completed_fs = row + .completed_flow_status + .as_ref() + .filter(|_| fs.is_none()) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let approval_module = raw_flow + .as_ref() + .zip(fs.as_ref().or(completed_fs.as_ref())) + .and_then(|(flow, status)| last_reached_approval_step(flow, status)); + let ac = run_ac.or_else(|| { + approval_module + .and_then(|module| module.suspend.as_ref()) + .and_then(approval_conditions_from_settings) + }); + let step = approval_module.map(|module| ApprovalStepView { + skin: module + .suspend + .as_ref() + .and_then(|s| s.skin) + .unwrap_or_default(), + summary: module.summary.clone().filter(|s| !s.trim().is_empty()), + }); // Fetch description, default_args, and enums from the step's completed job result let step_job_id = fs @@ -4988,9 +5175,12 @@ async fn get_approval_info( (None, None, None) }; - (form, desc, default_args, enums, ac, hc) + (form, desc, default_args, enums, ac, hc, step) }; + let skin = step.as_ref().map(|s| s.skin).unwrap_or_default(); + let step_summary = step.and_then(|s| s.summary); + let user_auth_required = approval_conditions .as_ref() .map(|ac| ac.user_auth_required) @@ -5020,6 +5210,9 @@ async fn get_approval_info( can_approve: false, user_auth_required, hide_cancel: None, + skin, + step_summary: None, + flow_summary: None, approvers: vec![], view_token: None, })); @@ -5055,6 +5248,9 @@ async fn get_approval_info( can_approve, user_auth_required, hide_cancel, + skin, + step_summary, + flow_summary: row.flow_summary, approvers, view_token, })) @@ -6470,7 +6666,7 @@ pub async fn run_flow_by_path( Query(run_query): Query, args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let (args, trigger_metadata) = get_args_and_trigger_metadata( + let (args, trigger_metadata) = match get_args_and_trigger_metadata( &db, &authed, RunnableId::from_flow_path(flow_path.to_path()), @@ -6478,7 +6674,15 @@ pub async fn run_flow_by_path( &w_id, args, ) - .await?; + .await? + { + WebhookRun::Run(args, trigger_metadata) => (args, trigger_metadata), + // 200 rather than an error: services drop a webhook that keeps failing, and disabling a + // trigger in Windmill must not cost it its registration. + WebhookRun::TriggerDisabled => { + return Ok((StatusCode::OK, NATIVE_TRIGGER_DISABLED_MSG.to_string())) + } + }; let (uuid, _, _, _) = push_flow_job_by_path_into_queue( authed, @@ -6903,7 +7107,7 @@ pub async fn run_script_by_path( Query(run_query): Query, args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let (args, trigger_metadata) = get_args_and_trigger_metadata( + let (args, trigger_metadata) = match get_args_and_trigger_metadata( &db, &authed, RunnableId::from_script_path(script_path.to_path()), @@ -6911,7 +7115,13 @@ pub async fn run_script_by_path( &w_id, args, ) - .await?; + .await? + { + WebhookRun::Run(args, trigger_metadata) => (args, trigger_metadata), + WebhookRun::TriggerDisabled => { + return Ok((StatusCode::OK, NATIVE_TRIGGER_DISABLED_MSG.to_string())) + } + }; let (uuid, _, _) = push_script_job_by_path_into_queue( authed, @@ -6929,6 +7139,16 @@ pub async fn run_script_by_path( Ok((StatusCode::CREATED, uuid.to_string())) } +/// What a webhook delivery resolved to: the arguments to run with, or nothing to run. +pub enum WebhookRun { + Run(PushArgsOwned, Option), + /// The native trigger this delivery belongs to is disabled. + TriggerDisabled, +} + +const NATIVE_TRIGGER_DISABLED_MSG: &str = + "This trigger is disabled in Windmill, so no job was created"; + #[allow(unused)] pub async fn get_args_and_trigger_metadata( db: &DB, @@ -6937,14 +7157,21 @@ pub async fn get_args_and_trigger_metadata( run_query: &RunJobQuery, w_id: &str, args: RawWebhookArgs, -) -> error::Result<(PushArgsOwned, Option)> { +) -> error::Result { use windmill_common::triggers::TriggerMetadata; // Build trigger metadata if this is a native trigger request #[cfg(feature = "native_trigger")] let (trigger_metadata, native_args) = if let Some(service_name_str) = &run_query.service_name { - use crate::native_triggers::{prepare_native_trigger_args, ServiceName}; + use crate::native_triggers::{ + native_trigger_is_enabled, prepare_native_trigger_args, ServiceName, + }; let service_name = ServiceName::try_from(service_name_str.to_owned())?; + if let Some(external_id) = run_query.trigger_external_id.as_deref() { + if !native_trigger_is_enabled(db, w_id, service_name, external_id).await? { + return Ok(WebhookRun::TriggerDisabled); + } + } let metadata = Some(TriggerMetadata::new( run_query.trigger_external_id.clone(), service_name.as_job_trigger_kind(), @@ -6980,7 +7207,7 @@ pub async fn get_args_and_trigger_metadata( .await? }; - Ok((args, trigger_metadata)) + Ok(WebhookRun::Run(args, trigger_metadata)) } #[derive(Deserialize)] @@ -7375,6 +7602,7 @@ async fn log_job_view( } pub async fn run_wait_result_job_by_path_get( + cross_site: CrossSiteGetGuard, method: hyper::http::Method, authed: ApiAuthed, Extension(user_db): Extension, @@ -7387,6 +7615,7 @@ pub async fn run_wait_result_job_by_path_get( check_license_key_valid().await?; let script_path = script_path.to_path(); + let runnable_id = cross_site.script_runnable(script_path)?; check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; if method == http::Method::HEAD { @@ -7399,12 +7628,7 @@ pub async fn run_wait_result_job_by_path_get( args.body = args::Body::HashMap(payload_as_args); let args = args - .to_args_from_runnable( - &db, - &w_id, - RunnableId::from_script_path(script_path), - run_query.skip_preprocessor, - ) + .to_args_from_runnable(&db, &w_id, runnable_id, run_query.skip_preprocessor) .await?; check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; @@ -7823,6 +8047,7 @@ pub async fn stream_flow_by_version( } pub async fn stream_script_by_path( + cross_site: CrossSiteGetGuard, authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, @@ -7831,12 +8056,13 @@ pub async fn stream_script_by_path( method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { + let runnable_id = cross_site.script_runnable(script_path.to_path())?; stream_job( authed, db, user_db, w_id, - RunnableId::from_script_path(script_path.to_path()), + runnable_id, args, run_query, method == http::Method::GET, @@ -8131,6 +8357,80 @@ pub async fn run_wait_result_flow_by_version( .await } +/// Whether request-supplied SQL from an operator may run. Operators can only run deployed +/// code, so a request their job token (`WM_TOKEN`) authenticates comes from code a +/// non-operator authored. The job must still be running, and the request must have the +/// shape `wmill.datatable()` sends (PostgreSQL against a `datatable://` database), so a +/// WM_TOKEN that leaked into job logs cannot be replayed to reach another target while the +/// job lives, in particular DuckDB, which runs in-process in the worker. +/// +/// What it does permit is any statement against the workspace's data tables, writes and DDL +/// included: the helper's body is an unrestricted SQL template. What that reaches is the +/// operator's own data table role — the preview job is permissioned as them, so the executor +/// resolves it under their tenancy like any other job. +/// +/// The database argument is only half the target: the executor honors a `-- database` +/// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused. +/// Check them against the code the executor runs rather than the request's `content`, which +/// is not the same string once a `WM_INTERNAL_DB` marker expands. +async fn operator_may_run_datatable_query( + db: &DB, + w_id: &str, + job_id: Option, + language: Option<&ScriptLang>, + content: &str, + args: Option<&HashMap>>, +) -> error::Result { + let Some(job_id) = job_id else { + return Ok(false); + }; + if language != Some(&ScriptLang::Postgresql) { + return Ok(false); + } + // Parse the directives out of the code the executor actually runs: it expands a + // `WM_INTERNAL_DB` marker first, and a directive can be embedded in the expansion. + // An expansion that overrides the language would run something other than the SQL the + // language check above cleared, so it is refused along with a malformed marker. + let executed = + match query_builders::try_expand_internal_db_query(content, &ScriptLang::Postgresql) { + Some(Ok(expanded)) if expanded.language_override.is_none() => Cow::Owned(expanded.code), + Some(_) => return Ok(false), + None => Cow::Borrowed(content), + }; + if windmill_parser_sql::parse_db_resource(&executed).is_some() + || !matches!(windmill_parser_sql::parse_s3_mode(&executed), Ok(None)) + { + return Ok(false); + } + let targets_datatable = args + .and_then(|args| args.get("database")) + .and_then(|database| serde_json::from_str::(database.get()).ok()) + .is_some_and(|database| database.starts_with("datatable://")); + if !targets_datatable { + return Ok(false); + } + Ok(sqlx::query_scalar!( + "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} + +/// The refusal an operator gets from a preview route. Inside a job the caller never ran a +/// preview themselves, so name the one thing the job's token may do. +fn operator_preview_refusal(job_id: Option) -> error::Error { + let reason = if job_id.is_some() { + "Operators cannot run preview jobs for security reasons: from inside a job, an \ + operator may only run a wmill.datatable() query while that job is running" + } else { + "Operators cannot run preview jobs for security reasons" + }; + error::Error::NotAuthorized(reason.to_string()) +} + async fn run_preview_script( authed: ApiAuthed, Extension(db): Extension, @@ -8142,9 +8442,20 @@ async fn run_preview_script( #[cfg(feature = "enterprise")] check_license_key_valid().await?; if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // A deferred run would outlive the running job the exemption keys off. + if run_query.get_scheduled_for(&db).await?.is_some() + || !operator_may_run_datatable_query( + &db, + &w_id, + authed.job_id, + preview.language.as_ref(), + preview.content.as_deref().unwrap_or_default(), + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(authed.job_id)); + } } // Preview runs arbitrary, request-supplied code. require_path_read_access_for_preview // only checks folder/namespace *read* access (and is a no-op when path is null), so a @@ -8239,13 +8550,21 @@ async fn run_inline_preview_script( Path(w_id): Path, Json(preview): Json, ) -> error::Result { - // Same arbitrary-code class as run_preview_script: operators are blocked from - // running request-supplied code, and a narrowly-scoped token must not escape - // its scope through inline preview. - if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // Same arbitrary-code class as run_preview_script, and every worker and standalone + // server exposes this route, so an operator is refused on the same terms. A + // narrowly-scoped token must not escape its scope through inline preview either. + if authed.is_operator + && !operator_may_run_datatable_query( + &db, + &w_id, + job_id, + Some(&preview.language), + &preview.content, + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(job_id)); } check_scopes(&authed, || format!("jobs:run"))?; if let Some(job_id) = job_id { @@ -8280,7 +8599,7 @@ async fn run_inline_preview_script( #[cfg(not(feature = "run_inline"))] async fn run_inline_preview_script() -> error::Result { Err(error::Error::InternalErr( - "inline preview requires the worker feature".to_string(), + "inline preview requires the run_inline feature on the worker".to_string(), )) } @@ -8404,7 +8723,12 @@ fn register_potential_assets_on_inline_execution( .as_ref() .and_then(|args| args.get("database")) .map(|v| v.get().trim_matches('"')) - .and_then(|dt| dt.strip_prefix("datatable://")); + .and_then(|dt| dt.strip_prefix("datatable://")) + // `?role=` picks the connection, not the data table. Anything else after a `?` may be + // part of a name stored before names were restricted, so it stays. + .map(|dt| { + windmill_common::workspaces::parse_datatable_ref(dt).map_or(dt, |(name, _)| name) + }); if let Some(datatable) = datatable { let re = regex::Regex::new(r#"SET search_path TO "([^"]+)";"#).unwrap(); let (schema, content) = if let Some(captures) = re.captures(&preview.content) { @@ -9244,7 +9568,7 @@ async fn run_preview_flow_job( .await?; // Set memory_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { + if let Some(memory_id) = run_query.memory_key(&w_id, &flow_path) { set_flow_memory_id(&mut tx, uuid, memory_id).await?; } @@ -9257,6 +9581,10 @@ async fn run_preview_flow_job( &flow_path, &run_query, user_message.as_ref(), + uuid, + // Run from the editor's test panel: a trial, not a real conversation. + true, + &flow_args, ) .await?; } @@ -11618,6 +11946,7 @@ mod approval_view_gate_tests { token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, } } @@ -11712,4 +12041,58 @@ mod approval_view_gate_tests { "trigger@example.com" )); } + + #[test] + fn approval_step_is_the_last_one_passed() { + let flow: FlowValue = serde_json::from_value(serde_json::json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, "suspend": {} }, + { "id": "b", "value": { "type": "identity" }, "suspend": {} }, + { "id": "c", "value": { "type": "identity" } } + ]})) + .unwrap(); + let step_at = |step: i32, types: [(&str, bool); 3]| { + let mut status = FlowStatus::new(&flow); + status.step = step; + status.modules = ["a", "b", "c"] + .into_iter() + .zip(types) + .map(|(id, (kind, skipped))| { + serde_json::from_value(serde_json::json!({ + "type": kind, "id": id, "job": Uuid::nil(), "count": 1, + "failed_retries": [], "skipped": skipped + })) + .unwrap() + }) + .collect(); + last_reached_approval_step(&flow, &status).map(|module| module.id.clone()) + }; + let waiting = ("WaitingForEvents", false); + let pending = ("WaitingForPriorSteps", false); + let ran = ("Success", false); + let skipped = ("Success", true); + // Awaiting a's approval: b, itself an approval step, already holds `WaitingForEvents`. + assert_eq!(step_at(1, [ran, waiting, pending]).as_deref(), Some("a")); + assert_eq!(step_at(2, [ran, ran, waiting]).as_deref(), Some("b")); + assert_eq!(step_at(3, [ran, skipped, ran]).as_deref(), Some("a")); + assert_eq!(step_at(0, [pending, pending, pending]), None); + } + + #[test] + fn approved_step_stays_gated_by_its_settings() { + let from_settings = |suspend: serde_json::Value| { + approval_conditions_from_settings(&serde_json::from_value(suspend).unwrap()) + }; + let login = from_settings(serde_json::json!({ + "user_auth_required": true, + "user_groups_required": { "type": "static", "value": ["approvers"] } + })); + assert!(!can_view( + &None, + &login, + Some("f/team/flow"), + "trigger@example.com" + )); + assert_eq!(login.unwrap().user_groups_required, ["approvers"]); + assert!(from_settings(serde_json::json!({})).is_none()); + } } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index de49949043..0fd5ac31b5 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -66,7 +66,14 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; -mod ai_skills; +#[cfg(feature = "private")] +mod ai_free_tier_ee; +mod ai_free_tier_oss; +#[cfg(feature = "parquet")] +mod ai_sessions; +#[cfg(feature = "parquet")] +pub use ai_sessions::sweep_expired_ai_session_backups; +mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; @@ -78,6 +85,7 @@ pub mod azure_proxy_ee; mod azure_proxy_oss; mod capture; mod concurrency_groups; +mod csrf; mod db; mod db_health; mod dbt; @@ -376,6 +384,7 @@ async fn inject_agent_authed( token_prefix: None, read_only: false, job_id: None, + credential_expiry: None, }, job_id: None, }); @@ -450,15 +459,15 @@ pub async fn run_server( // unless they are allowed — hence a separate layer rather than widening the // one every other route shares. (`Mcp-Param-*` is only sent for tool inputs // annotated with `x-mcp-header`, which no tool here declares.) + // + // The request's own header list is mirrored rather than enumerated: a browser + // MCP client may send any custom name for a preprocessor to read, and no fixed + // list could cover them. Nothing is granted by echoing it: the origin is + // `Any`, so browsers never attach credentials, and the endpoint authenticates + // each request on its own. let mcp_cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST, http::Method::DELETE]) - .allow_headers([ - http::header::CONTENT_TYPE, - http::header::AUTHORIZATION, - http::HeaderName::from_static("mcp-protocol-version"), - http::HeaderName::from_static("mcp-method"), - http::HeaderName::from_static("mcp-name"), - ]) + .allow_headers(tower_http::cors::AllowHeaders::mirror_request()) // The 401 challenge is how a client discovers where to authorize (RFC 9728), // and it is not a safelisted response header, so without this a browser // client sees an empty one and has no way to begin the OAuth flow. @@ -551,7 +560,7 @@ pub async fn run_server( if server_mode || mcp_mode { use mcp::{ add_www_authenticate_header, add_www_authenticate_header_gateway, - extract_workspace_from_token, + extract_workspace_from_token, reject_token_query_param, }; let (mcp_router, mcp_cancellation_token) = setup_mcp_server( db.clone(), @@ -564,15 +573,17 @@ pub async fn run_server( let workspaced_mcp_router = mcp_router .clone() .route_layer(from_extractor::()) + .layer(axum::middleware::from_fn(reject_token_query_param)) .layer(axum::middleware::from_fn(add_www_authenticate_header)) .layer(axum::middleware::from_fn(extract_and_store_workspace_id)); // Gateway MCP router — resolves workspace from token let gateway_mcp_router = mcp_router .route_layer(from_extractor::()) + .layer(axum::middleware::from_fn(extract_workspace_from_token)) + .layer(axum::middleware::from_fn(reject_token_query_param)) .layer(axum::middleware::from_fn( add_www_authenticate_header_gateway, - )) - .layer(axum::middleware::from_fn(extract_workspace_from_token)); + )); ( workspaced_mcp_router, gateway_mcp_router, @@ -655,9 +666,13 @@ pub async fn run_server( "/workspace_dependencies", workspace_dependencies::workspaced_service(), ) + // CORS so a chat UI on another origin (an external site, or + // a sandboxed raw app with its frontend SDK token) can read + // its conversation history. Bearer-only, like variables. .nest( "/flow_conversations", - windmill_api_flow_conversations::workspaced_service(), + windmill_api_flow_conversations::workspaced_service() + .layer(cors.clone()), ) // CORS so an opaque-origin app iframe (WIN-2006 embed, // no separate domain) can read folders/listnames with a @@ -710,7 +725,6 @@ pub async fn run_server( Router::new() }) .nest("/ai", ai::workspaced_service()) - .nest("/ai_skills", ai_skills::workspaced_service()) .nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service()) .nest( "/path_autocomplete", @@ -957,6 +971,15 @@ pub async fn run_server( #[cfg(not(feature = "enterprise"))] Router::new() }) + .nest("/w/{workspace_id}/git_sync", { + #[cfg(feature = "enterprise")] + { + git_sync_oss::workspaced_git_sync_service() + } + + #[cfg(not(feature = "enterprise"))] + Router::new() + }) .nest( "/w/{workspace_id}/resources_u", public_service().layer(cors.clone()), diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 4c6f5e245c..334ee340a8 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -6,8 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ -use sqlx::Postgres; -use windmill_common::error::Error; +use sqlx::{PgConnection, Postgres}; +use tokio::task::JoinHandle; +use windmill_common::{db::DB, error::Error}; use crate::db::CustomMigrator; use sqlx::migrate::Migrate; @@ -121,3 +122,294 @@ async fn fix_flow_versioning_migration(migrator: &mut CustomMigrator) -> Result< } Ok(()) } + +// Held for the whole background run so only one server does it at a time. +const BACKGROUND_MIGRATIONS_LOCK_ID: i64 = 4_931_072_518_336_401; + +/// Schema changes too slow to hold server startup for, run by one server at a time after the +/// sqlx migrations. Each step is recorded in `windmill_migrations` once done; a step interrupted +/// by a restart or an error starts over on the next start and must resume safely. +pub fn spawn_background_migrations( + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> JoinHandle<()> { + tokio::spawn(async move { + tokio::select! { + r = run_background_migrations(&db) => { + if let Err(err) = r { + tracing::error!("Background migrations stopped, retrying on the next start: {err:#}"); + } + } + _ = killpill_rx.recv() => { + tracing::info!("Killpill received, stopping background migrations"); + } + } + }) +} + +async fn run_background_migrations(db: &DB) -> Result<(), Error> { + // Detached so the pool's 5min statement_timeout, lifted here for the index builds, never + // comes back with this connection; closing it also releases the advisory lock. + let mut conn = db.acquire().await?.detach(); + conn.execute("SET statement_timeout = 0").await?; + let locked = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)") + .bind(BACKGROUND_MIGRATIONS_LOCK_ID) + .fetch_one(&mut conn) + .await?; + if !locked { + return Ok(()); + } + + for step in [AUDIT_OPERATION_INDEX, RETIRE_LEGACY_AUDIT] { + if background_migration_done(&mut conn, step).await? { + continue; + } + // Steps bound the lock waits that would queue audit inserts behind them with + // lock_timeout, and resume where they stopped, so a lock timeout is retried here. + let mut attempt = 1; + loop { + let run = match step { + AUDIT_OPERATION_INDEX => create_audit_operation_index(&mut conn).await, + RETIRE_LEGACY_AUDIT => retire_legacy_audit_table(&mut conn).await, + _ => unreachable!("background migration {step} has no step function"), + }; + match run { + Ok(()) => break, + Err(err) if attempt < 10 && is_lock_timeout(&err) => { + tracing::warn!( + "Background migration {step} timed out on a lock, retrying in 30s: {err:#}" + ); + attempt += 1; + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + } + Err(err) => return Err(err), + } + } + mark_background_migration_done(&mut conn, step).await?; + } + Ok(()) +} + +const AUDIT_OPERATION_INDEX: &str = "audit_partitioned_workspace_operation_index"; +const RETIRE_LEGACY_AUDIT: &str = "retire_legacy_audit_table"; + +// Short on purpose: a statement waiting for its lock is also a wait for every audit insert +// queued behind it. +const STEP_LOCK_TIMEOUT: &str = "SET lock_timeout = '1s'"; + +fn is_lock_timeout(err: &Error) -> bool { + matches!(err, Error::SqlErr { error: sqlx::Error::Database(db_err), .. } + if db_err.code().as_deref() == Some("55P03")) +} + +async fn background_migration_done(conn: &mut PgConnection, name: &str) -> Result { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM windmill_migrations WHERE name = $1)", + ) + .bind(name) + .fetch_one(conn) + .await?) +} + +async fn mark_background_migration_done(conn: &mut PgConnection, name: &str) -> Result<(), Error> { + sqlx::query("INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING") + .bind(name) + .execute(conn) + .await?; + tracing::info!("Background migration {name} done"); + Ok(()) +} + +const AUDIT_OPERATION_INDEX_KEY: &str = r#"(workspace_id, operation, id DESC, "timestamp")"#; + +/// A plain `CREATE INDEX` on the partitioned table holds a SHARE lock on every partition until the +/// whole build ends, blocking the audit insert each job push makes in its own transaction. Each +/// partition is built CONCURRENTLY instead and attached to a parent created `ON ONLY`, which turns +/// valid once all partitions are attached. Partitions created later get the index from the parent. +async fn create_audit_operation_index(conn: &mut PgConnection) -> Result<(), Error> { + // Metadata only, but it waits for every open transaction that wrote an audit row, and every + // audit insert (so every job push) queues behind it meanwhile. Give up early instead. + conn.execute(STEP_LOCK_TIMEOUT).await?; + let created = conn + .execute( + format!( + "CREATE INDEX IF NOT EXISTS ix_audit_partitioned_workspace_operation \ + ON ONLY audit_partitioned {AUDIT_OPERATION_INDEX_KEY}" + ) + .as_str(), + ) + .await; + conn.execute("RESET lock_timeout").await?; + created?; + let partitions: Vec = sqlx::query_scalar( + "SELECT c.relname::text FROM pg_inherits p JOIN pg_class c ON c.oid = p.inhrelid + WHERE p.inhparent = 'audit_partitioned'::regclass + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits ip JOIN pg_index i ON i.indexrelid = ip.inhrelid + WHERE ip.inhparent = 'ix_audit_partitioned_workspace_operation'::regclass + AND i.indrelid = c.oid) + ORDER BY c.relname DESC", + ) + .fetch_all(&mut *conn) + .await?; + let quote = |name: &str| format!("\"{}\"", name.replace('"', "\"\"")); + for partition in partitions { + let index = quote(&format!( + "{partition}_workspace_id_operation_id_timestamp_idx" + )); + tracing::info!("Building ix_audit_partitioned_workspace_operation on {partition}"); + // An interrupted CONCURRENTLY build leaves an invalid index under this name. + conn.execute(format!("DROP INDEX CONCURRENTLY IF EXISTS {index}").as_str()) + .await?; + conn.execute( + format!( + "CREATE INDEX CONCURRENTLY {index} ON {} {AUDIT_OPERATION_INDEX_KEY}", + quote(&partition) + ) + .as_str(), + ) + .await?; + conn.execute( + format!( + "ALTER INDEX ix_audit_partitioned_workspace_operation ATTACH PARTITION {index}" + ) + .as_str(), + ) + .await?; + } + Ok(()) +} + +const AUDIT_COLUMNS: &str = + "workspace_id, id, timestamp, username, operation, action_kind, resource, parameters, email, span"; + +/// Moves the last 30 days of the pre-partitioning `audit` table into daily partitions and drops +/// it with anything older. An empty `audit` view takes its place for servers still running an +/// older version, which read it in a `UNION ALL` with `audit_partitioned`. +async fn retire_legacy_audit_table(conn: &mut PgConnection) -> Result<(), Error> { + let is_table = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_class WHERE oid = to_regclass('audit') AND relkind = 'r')", + ) + .fetch_one(&mut *conn) + .await?; + if !is_table { + return Ok(()); + } + // Creating a partition and re-owning the sequence queue every audit insert (so every job + // push) behind them while they wait for their own lock, e.g. on a long-running reader, and + // the DROP queues older servers' reads the same way. Give up early and retry instead. + conn.execute(STEP_LOCK_TIMEOUT).await?; + + let days: Vec = sqlx::query_scalar( + "SELECT DISTINCT timestamp::date FROM audit + WHERE timestamp > now() - interval '30 days' ORDER BY 1", + ) + .fetch_all(&mut *conn) + .await?; + for day in days { + let next = day + chrono::Duration::days(1); + // Created outside any transaction: a partition created inside one keeps a lock on + // audit_partitioned that blocks every job push's audit insert until it commits. + conn.execute( + format!( + "CREATE TABLE IF NOT EXISTS \"audit_{}\" PARTITION OF audit_partitioned \ + FOR VALUES FROM ('{day}') TO ('{next}')", + day.format("%Y%m%d") + ) + .as_str(), + ) + .await?; + // One statement per day, so a row is always in exactly one of the two tables. + let moved = sqlx::query(&format!( + "WITH moved AS ( + DELETE FROM audit WHERE timestamp >= $1::date AND timestamp < $1::date + 1 + RETURNING {AUDIT_COLUMNS} + ) + INSERT INTO audit_partitioned ({AUDIT_COLUMNS}) SELECT {AUDIT_COLUMNS} FROM moved" + )) + .bind(day) + .execute(&mut *conn) + .await?; + tracing::info!( + "Moved {} legacy audit rows of {day} into audit_partitioned", + moved.rows_affected() + ); + } + + // audit_partitioned draws its ids from this sequence, which dropping its owner would drop. + // Not in the transaction below: its lock stops every insert's nextval until commit, and the + // DROP can wait there on readers of the old table. + conn.execute("ALTER SEQUENCE audit_id_seq OWNED BY audit_partitioned.id") + .await?; + let mut tx = conn.begin().await?; + // Servers older than audit partitioning still write to the table during a rolling upgrade: + // rows they added after their day was moved above are swept up under the lock. + tx.execute("LOCK TABLE audit IN ACCESS EXCLUSIVE MODE") + .await?; + tx.execute( + format!( + "WITH moved AS ( + DELETE FROM audit WHERE timestamp > now() - interval '30 days' + RETURNING {AUDIT_COLUMNS} + ) + INSERT INTO audit_partitioned ({AUDIT_COLUMNS}) SELECT {AUDIT_COLUMNS} FROM moved" + ) + .as_str(), + ) + .await?; + tx.execute("DROP TABLE audit").await?; + tx.execute( + format!("CREATE VIEW audit AS SELECT {AUDIT_COLUMNS} FROM audit_partitioned WHERE false") + .as_str(), + ) + .await?; + tx.execute("GRANT ALL ON audit TO windmill_user, windmill_admin") + .await?; + tx.commit().await?; + conn.execute("RESET lock_timeout").await?; + tracing::info!("Retired the legacy audit table"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_audit_table_keeps_the_last_30_days(db: DB) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO audit (workspace_id, id, timestamp, username, operation, action_kind) + VALUES ('w', -2, now() - interval '3 days', 'u', 'recent', 'execute'), + ('w', -1, now() - interval '60 days', 'u', 'old', 'execute')", + ) + .execute(&db) + .await?; + + retire_legacy_audit_table(&mut *db.acquire().await?).await?; + + let moved: Vec<(i64, String)> = sqlx::query_as( + "SELECT id, operation::text FROM audit_partitioned WHERE workspace_id = 'w'", + ) + .fetch_all(&db) + .await?; + assert_eq!(moved, vec![(-2, "recent".to_string())]); + + // The id sequence outlives the table that owned it. + sqlx::query( + "INSERT INTO audit_partitioned (workspace_id, username, operation, action_kind) + VALUES ('w', 'u', 'after', 'execute')", + ) + .execute(&db) + .await?; + + // Servers on an older version still read `audit` in a UNION with audit_partitioned. + let seen: i64 = sqlx::query_scalar( + "SELECT count(*) FROM (SELECT * FROM audit_partitioned UNION ALL SELECT * FROM audit) a + WHERE workspace_id = 'w'", + ) + .fetch_one(&db) + .await?; + assert_eq!(seen, 2); + Ok(()) + } +} diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 3f89660fc4..ca78f80328 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -889,7 +889,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runScriptByPath"), description: Cow::Borrowed("run script by path"), - instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/p/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ @@ -1276,13 +1276,15 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1293,7 +1295,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } } @@ -1391,13 +1393,15 @@ is, a different one moves it there and archives the old path"), }, "execution_mode": { "type": "string", - "description": "Who the app's runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Either way `anonymous`, which makes the app publicly executable, is never assumed. Possible values: viewer, publisher, anonymous" + "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1408,7 +1412,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } }, @@ -1430,7 +1434,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runFlowByPath"), description: Cow::Borrowed("run flow by path"), - instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/f/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 2b4350bcfe..86fbec1806 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -12,7 +12,9 @@ use windmill_mcp::common::transform::transform_property_keys; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend, PathFilter}; +use windmill_mcp::server::{ + BackendResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter, +}; use crate::auth::AuthCache; use crate::db::ApiAuthed; @@ -41,11 +43,14 @@ use axum::{ extract::{Extension, Path}, http::Request, middleware::Next, - response::Response, + response::{IntoResponse, Response}, routing::get, Json, Router, }; -use windmill_common::{auth::hash_token, db::GatewayWorkspaceId, error::JsonResult}; +use windmill_common::{ + auth::hash_token, db::GatewayWorkspaceId, error::JsonResult, + global_settings::MCP_DISABLE_TOKEN_QUERY_PARAM, +}; // McpAuth impl for ApiAuthed is in windmill-api-auth (same crate as the type) @@ -214,8 +219,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, false, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_script_by_path_internal( self.db.clone(), @@ -238,8 +246,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, true, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_flow_by_path_internal( self.db.clone(), @@ -438,6 +449,29 @@ pub async fn add_www_authenticate_header( } } +/// Middleware refusing a credential carried in the MCP URL once the instance sets +/// `mcp_disable_token_query_param`. Sits outside everything that reads the token, so neither +/// the gateway lookup nor `ApiAuthed` ever sees it, and inside the `WWW-Authenticate` layer, +/// whose header is what sends the client into the OAuth flow instead. Refused rather than +/// ignored: the URL leaked the token whether or not the request used it. +pub async fn reject_token_query_param(request: Request, next: Next) -> Response { + let carries_token = MCP_DISABLE_TOKEN_QUERY_PARAM.load(std::sync::atomic::Ordering::Relaxed) + && request + .uri() + .query() + .is_some_and(|q| url::form_urlencoded::parse(q.as_bytes()).any(|(k, _)| k == "token")); + if carries_token { + return ( + axum::http::StatusCode::UNAUTHORIZED, + "This instance does not accept a token in the MCP URL. Remove the token query \ + parameter and let your client sign in through OAuth, or send the token in an \ + Authorization header.", + ) + .into_response(); + } + next.run(request).await +} + /// Extract the bearer token from either the `Authorization` header or the /// `?token=` query parameter (MCP clients commonly pass it in the URL). fn extract_gateway_token(request: &Request) -> Option { diff --git a/backend/windmill-api/src/mcp/mod.rs b/backend/windmill-api/src/mcp/mod.rs index 5f6bd5edb5..5545d59a9f 100644 --- a/backend/windmill-api/src/mcp/mod.rs +++ b/backend/windmill-api/src/mcp/mod.rs @@ -12,5 +12,5 @@ pub mod oauth_server; pub use core::{ add_www_authenticate_header, add_www_authenticate_header_gateway, extract_and_store_workspace_id, extract_workspace_from_token, list_tools_service, - setup_mcp_server, + reject_token_query_param, setup_mcp_server, }; diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 94271bb662..ce5cb478cf 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -11,15 +11,19 @@ use serde_json::Value; use sql_builder::prelude::*; use windmill_common::auth::create_jwt_token; use windmill_common::db::{Authed, UserDB}; +use windmill_common::error::Error; use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; +use windmill_common::triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}; use windmill_common::utils::{query_elems_from_hub, StripPath}; use windmill_common::worker::to_raw_value; use windmill_common::{DB, HUB_BASE_URL}; use windmill_mcp::server::{ - non_empty_body_fields, BackendResult, EndpointTool, ErrorData, PathFilter, + non_empty_body_fields, BackendResult, EndpointTool, ErrorData, McpRequest, PathFilter, }; use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; +use windmill_trigger::trigger_helpers::{get_runnable_format, RunnableId}; +use crate::args::build_headers; use crate::db::ApiAuthed; use crate::HTTP_CLIENT; @@ -641,7 +645,7 @@ fn selects_endpoint_tool(caller_scopes: &[String], tool: &str) -> bool { .is_ok_and(|config| config.endpoints.iter().any(|e| e == tool)) } -/// Create HTTP request with authentication +/// Create HTTP request with authentication. pub async fn create_http_request( method: &str, url: &str, @@ -702,17 +706,113 @@ pub async fn create_http_request( .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) } -/// Convert a JSON Value into PushArgsOwned for job execution -pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned { +/// The `kind` an MCP-invoked runnable sees on its preprocessor event, alongside +/// `webhook`, `http` and the trigger kinds. +const MCP_TRIGGER_KEY: &str = "mcp"; + +/// A preprocessor's view of the MCP request that ran it. Mirrors the HTTP +/// trigger event: `body` is what the model sent, everything else describes the +/// call itself. +#[derive(serde::Serialize)] +struct McpPreprocessorEvent<'a> { + kind: &'a str, + body: Box, + headers: HashMap>, + tool_name: &'a str, +} + +/// Headers withheld from a preprocessor because they authenticate the connection. +/// +/// Not a security boundary: a webhook preprocessor receives all three. Withheld +/// because nothing needs them yet, and releasing one later is additive while +/// withdrawing one after runnables read it is not. +const WITHHELD_FROM_PREPROCESSOR: &[&str] = &["authorization", "cookie", "proxy-authorization"]; + +/// Every header a preprocessor may see. +fn preprocessor_headers( + headers: &http::HeaderMap, +) -> HashMap> { + let mut selected = build_headers(headers, None, true); + selected.retain(|name, _| { + !WITHHELD_FROM_PREPROCESSOR + .iter() + .any(|withheld| withheld.eq_ignore_ascii_case(name)) + }); + selected +} + +/// Build the job arguments for a script or flow run as an MCP tool. +/// +/// Shaped by the runnable's own format: a preprocessor receives the request as +/// an event, and a runnable without one receives only what the model sent. +pub async fn prepare_push_args( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + args: Value, + request: &McpRequest<'_>, +) -> Result { + let mut main_args = HashMap::new(); if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); for (k, v) in map { - args_hash.insert(k, to_raw_value(&v)); + main_args.insert(k, to_raw_value(&v)); } - windmill_queue::PushArgsOwned { extra: None, args: args_hash } - } else { - windmill_queue::PushArgsOwned::default() } + + let runnable_id = if is_flow { + RunnableId::from_flow_path(path) + } else { + // Resolves a `hub/` path to the hub script on its own. + RunnableId::from_script_path(path) + }; + + // MCP is not one of the `TRIGGER_KIND` enum values and does not need to be: + // the per-kind arms of the no-preprocessor heuristic are payload-shape + // special cases for message triggers, and `Webhook` reaches the same generic + // arm MCP wants while sharing that kind's format cache. + let runnable_format = get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; + + Ok(match runnable_format { + // Without a preprocessor there is nowhere for a header to go that the + // model does not also write: its arguments *are* the runnable's + // parameters, so a header bound to one of them would be a value the model + // could set. The request is reachable through a preprocessor, where it + // arrives in a key of the event the model never fills. + RunnableFormat { has_preprocessor: false, .. } => { + windmill_queue::PushArgsOwned { args: main_args, extra: None } + } + RunnableFormat { has_preprocessor: true, version } => { + let headers = preprocessor_headers(request.headers); + match version { + RunnableFormatVersion::V2 => { + let event = McpPreprocessorEvent { + kind: MCP_TRIGGER_KEY, + body: to_raw_value(&main_args), + headers, + tool_name: request.tool_name, + }; + windmill_queue::PushArgsOwned { + args: HashMap::from([("event".to_string(), to_raw_value(&event))]), + extra: None, + } + } + RunnableFormatVersion::V1 => windmill_queue::PushArgsOwned { + args: main_args, + extra: Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": MCP_TRIGGER_KEY, + MCP_TRIGGER_KEY: { + "headers": headers, + "tool_name": request.tool_name, + } + })), + )])), + }, + } + } + }) } /// Parse an HTTP response body into a JSON Value @@ -1321,6 +1421,7 @@ mod tests { token_prefix: None, read_only: false, job_id, + credential_expiry: None, } } diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index a69441ea75..26eb06c8e8 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -128,6 +128,17 @@ struct WorkspaceReassignment { // ---- Preview helpers ---- +/// The principal a departing member's runnables run as, or `None` when none of them are theirs +/// to hand over. `usr.username` is constrained to `[\w-]+`, so a member is `u/{username}`, except +/// a legacy `group-*` username, which canonicalizes to the group it names: what runs under that +/// principal runs as the group, which outlives the member. `None` binds NULL, which the +/// `on_behalf_of = $n` queries then match nowhere. +fn departing_principal(username: &str) -> Option { + use windmill_common::users::{username_to_permissioned_as, PERMISSIONED_AS_GROUP_PREFIX}; + let principal = username_to_permissioned_as(username); + (!principal.starts_with(PERMISSIONED_AS_GROUP_PREFIX)).then_some(principal) +} + async fn get_offboard_preview( db: impl sqlx::PgExecutor<'_> + Copy, w_id: &str, @@ -136,10 +147,8 @@ async fn get_offboard_preview( ) -> Result { let user_prefix = format!("u/{}/%", username); let user_owner = format!("u/{}", username); - // Same form the mutation reassigns, so preview and execution cannot disagree. `usr.username` - // is constrained to `[\w-]+`, so a member is always named `u/{username}` — the address form a - // principal can also take names an account with no `usr` row, which is nobody offboardable. - let departing = windmill_common::users::username_to_permissioned_as(username); + // Same form the mutation reassigns, so preview and execution cannot disagree. + let departing = departing_principal(username); // ---- Owned objects (under u/{username}/) ---- let scripts = sqlx::query_scalar!( @@ -244,17 +253,17 @@ async fn get_offboard_preview( // ---- Operator references (not under user's path) ---- let obo_scripts = sqlx::query_scalar!( "SELECT path FROM script WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived AND NOT deleted", - &departing, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_flows = sqlx::query_scalar!( "SELECT path FROM flow WHERE on_behalf_of = $1 AND NOT path LIKE $2 AND workspace_id = $3 AND NOT archived", - &departing, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_apps = sqlx::query_scalar!( "SELECT path FROM app WHERE policy->>'on_behalf_of' = $1 AND NOT path LIKE $2 AND workspace_id = $3", - &user_owner, &user_prefix, w_id + departing.as_deref(), &user_prefix, w_id ).fetch_all(db).await?; let obo_schedules = sqlx::query_scalar!( @@ -618,6 +627,7 @@ pub(crate) async fn offboard_global_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email).await?; sqlx::query!("DELETE FROM workspace_invite WHERE email = $1", &email) .execute(&mut *tx) .await?; @@ -830,7 +840,7 @@ async fn offboard_user_from_workspace<'c>( new_permissioned_as: &str, ) -> Result { let new_prefix = reassign_to.to_string(); - let departing = windmill_common::users::username_to_permissioned_as(username); + let departing = departing_principal(username); // The app policy stores an address beside its principal, and script/flow keep one for the // workers that still read it, so the replacement's is resolved here. @@ -870,7 +880,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( "UPDATE script SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -911,7 +921,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( "UPDATE flow SET on_behalf_of = $1, on_behalf_of_email = $4 WHERE on_behalf_of = $2 AND workspace_id = $3", new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -924,7 +934,7 @@ async fn offboard_user_from_workspace<'c>( sqlx::query!( r#"UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('script', 'flow') AND value->>'on_behalf_of' = $2 AND workspace_id = $3"#, new_permissioned_as, - &departing, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) @@ -950,9 +960,21 @@ async fn offboard_user_from_workspace<'c>( "UPDATE app SET policy = jsonb_set( jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)), ARRAY['on_behalf_of_email'], to_jsonb($4::text) - ) WHERE policy->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + ) WHERE policy->>'on_behalf_of' = $2 AND workspace_id = $3", &new_permissioned_as, - username, + departing.as_deref(), + w_id, + new_on_behalf_of_user_email + ) + .execute(&mut **tx) + .await?; + + // An app draft carries a copy of the deployed policy and is deployed from it, so it needs + // the same pair rewritten — the draft sweep above only covers scripts and flows. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text)), ARRAY['policy', 'on_behalf_of_email'], to_jsonb($4::text))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = $2 AND workspace_id = $3"#, + new_permissioned_as, + departing.as_deref(), w_id, new_on_behalf_of_user_email ) diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index c145252251..43ef23aff2 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -116,6 +116,10 @@ struct RunnableItem { use_codebase: Option, #[serde(skip_serializing_if = "Option::is_none")] has_deploy_errors: Option, + // flow-only: projected from the value so the home list can badge flows that + // open as a chat without fetching every flow's value. + #[serde(skip_serializing_if = "Option::is_none")] + chat_input_enabled: Option, // app-only #[serde(skip_serializing_if = "Option::is_none")] raw_app: Option, @@ -261,7 +265,8 @@ fn branch_sqls() -> Branches { FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred}) as draft_users" + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred} \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users" ) }; @@ -273,7 +278,7 @@ fn branch_sqls() -> Branches { o.ws_error_handler_muted, o.created_at as edited_at, \ o.hash, o.language::text as language, o.kind::text as script_kind, o.auto_kind, \ o.codebase IS NOT NULL as use_codebase, \ - (o.lock_error_logs IS NOT NULL) as has_deploy_errors, \ + (o.lock_error_logs IS NOT NULL) as has_deploy_errors, NULL::bool as chat_input_enabled, \ NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, o.hash as tiebreak \ FROM script o \ @@ -290,6 +295,7 @@ fn branch_sqls() -> Branches { o.ws_error_handler_muted, o.edited_at, \ NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \ + (o.value->>'chat_input_enabled')::bool as chat_input_enabled, \ NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ o.edited_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ FROM flow o \ @@ -305,7 +311,7 @@ fn branch_sqls() -> Branches { {draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \ NULL::bool as ws_error_handler_muted, av.created_at as edited_at, \ NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ - NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \ + NULL::bool as use_codebase, NULL::bool as has_deploy_errors, NULL::bool as chat_input_enabled, \ av.raw_app, o.policy->>'execution_mode' as execution_mode, o.id, \ o.versions[array_upper(o.versions, 1)] as version, \ COALESCE(av.created_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ @@ -343,11 +349,21 @@ fn draft_branch_sql(kind: &str) -> String { let kind_cols = match kind { "script" => { "d.value->>'language' as language, d.value->>'kind' as script_kind, \ - d.value->>'auto_kind' as auto_kind, false as raw_app" + d.value->>'auto_kind' as auto_kind, false as raw_app, \ + NULL::bool as chat_input_enabled" + } + // Type-guarded rather than a bare `::bool` cast: draft JSON is stored + // unvalidated, so a malformed value must yield NULL, not abort the list, + // and the string "true" must not count as enabled. + "flow" => { + "NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ + false as raw_app, \ + CASE WHEN json_typeof(d.value->'value'->'chat_input_enabled') = 'boolean' \ + THEN (d.value->'value'->>'chat_input_enabled')::bool END as chat_input_enabled" } _ => { "NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ - (d.typ = 'raw_app') as raw_app" + (d.typ = 'raw_app') as raw_app, NULL::bool as chat_input_enabled" } }; format!( @@ -358,7 +374,7 @@ fn draft_branch_sql(kind: &str) -> String { NULL::text[] as labels, NULL::text[] as inherited_labels, \ NULL::bool as ws_error_handler_muted, o.created_at as edited_at, \ NULL::bigint as hash, o.language, o.script_kind, o.auto_kind, \ - NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \ + NULL::bool as use_codebase, NULL::bool as has_deploy_errors, o.chat_input_enabled, \ o.raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.draft_path, o.path)) as sort_name, 0::bigint as tiebreak \ FROM ( \ diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 99dc7c41a8..a54f275f46 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -88,6 +88,66 @@ async fn list_files( Ok(Json(rows)) } +/// Rebuild one source log file from the columnar store. +/// +/// Not the original bytes: the store holds a line's fields rather than its text, +/// so the JSON is re-serialized here and key order and whitespace are this +/// writer's. Everything a reader can see survives — the drawer this feeds +/// renders a prettified view of each line either way, and a line that was never +/// JSON comes back exactly as it was written. +#[cfg(all(feature = "tantivy", feature = "private"))] +async fn get_log_file_from_store( + db: &DB, + store: &windmill_indexer::service_logs_store_ee::Store, + path: &str, +) -> windmill_common::error::Result { + let (hostname, file_name) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path".to_string()))?; + + // The store is partitioned by day and mode, neither of which the path + // carries. `log_file` names both, and its primary key starts with hostname. + let file = sqlx::query!( + // `mode!` because the column is NOT NULL and only the cast makes sqlx + // think otherwise; a silent default would look up a `mode=` partition + // that matches nothing and read as a missing file. + "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + hostname, + file_name + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + // A row registered by this version carries the minute in the file's own name, + // so the two agree and the second is redundant. One written before the + // uploader derived `log_ts` from the name carries a wall clock instead, and + // those outlive an upgrade by the retention period — which is also what makes + // the `ORDER BY` above worth having. The name is authoritative, so both go. + let mut known_ts = vec![chrono::DateTime::from_naive_utc_and_offset( + file.log_ts, + chrono::Utc, + )]; + if let Some(named) = file_name.rsplit('.').next().and_then(|s| { + chrono::NaiveDateTime::parse_from_str(s, windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .ok() + }) { + known_ts.push(chrono::DateTime::from_naive_utc_and_offset( + named, + chrono::Utc, + )); + } + + let text = windmill_indexer::service_logs_store_ee::read_log_file( + store, &file.mode, hostname, file_name, &known_ts, + ) + .await + .map_err(|e| Error::internal_err(format!("Error reading the service log store: {e}")))? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + Ok(content_plain(Body::from(text))) +} + async fn get_log_file( authed: ApiAuthed, Extension(db): Extension, @@ -104,27 +164,30 @@ async fn get_log_file( let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { - let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client + use windmill_object_store::object_store_reexports::ObjectStoreError; + + // The raw file, for as long as it is there. It outlives its ingestion by + // one indexer pass at most, so this covers the most recent minutes of a + // host's logs byte for byte; everything older is rebuilt from the store. + let object_path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); + match s3_client .get(&windmill_object_store::object_store_reexports::Path::from( - path, + object_path, )) - .await; - match file { - Ok(file) => { - let bytes = file.bytes().await; - match bytes { - Ok(bytes) => { - return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); - } - Err(e) => { - return Err(Error::internal_err(format!( - "Error pulling the bytes: {}", - e - ))); - } + .await + { + Ok(file) => match file.bytes().await { + Ok(bytes) => { + return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); } - } + Err(e) => { + return Err(Error::internal_err(format!( + "Error pulling the bytes: {}", + e + ))); + } + }, + Err(ObjectStoreError::NotFound { .. }) => {} Err(e) => { return Err(Error::internal_err(format!( "Error fetching the file: {}", @@ -132,6 +195,11 @@ async fn get_log_file( ))); } } + + #[cfg(all(feature = "tantivy", feature = "private"))] + return get_log_file_from_store(&db, &s3_client, &path).await; + #[cfg(not(all(feature = "tantivy", feature = "private")))] + return Err(Error::NotFound(format!("File {path} not found"))); } let full_path = format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path); // SECURITY (defense in depth): refuse to read through a symlink so a planted diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 593ea3b298..204b30010d 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -13,19 +13,25 @@ use sha2::Sha256; use sqlx::types::Uuid; use std::collections::HashMap; use windmill_common::error::{to_anyhow, Error}; +use windmill_common::flows::ApprovalSkin; +use windmill_common::utils::truncate_with_ellipsis; use windmill_common::variables::{get_secret_value_as_admin, get_workspace_key}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; use crate::{ approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryButtonText, QueryDefaultArgsJson, QueryDynamicEnumJson, - QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, + extract_w_id_from_resume_url, get_approval_step_skin, handle_resume_action, + ApprovalFormDetails, FieldType, MessageFormat, QueryButtonText, QueryDefaultArgsJson, + QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, }, auth::OptTokened, }; +// Slack rejects a button value over 2000 characters, and with it the whole post. The button value +// carries the message on to the modal, so the message is shortened to fit. +const SLACK_BUTTON_VALUE_MAX_CHARS: usize = 2000; + #[derive(Deserialize, Debug)] pub struct SlackFormData { payload: String, @@ -127,6 +133,9 @@ struct PrivateMetadata { // HMAC over (w_id, resource_path) keyed on the workspace key; minted when the modal is // built, required by `handle_submission` before the resource_path is decrypted. signature: Option, + // Only selects the wording of the updated channel message, so it is left unsigned. + #[serde(default)] + skin: ApprovalSkin, } // Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify @@ -432,6 +441,7 @@ async fn handle_submission( let container: Container = private_metadata.container; let hide_cancel = private_metadata.hide_cancel; let signature = private_metadata.signature; + let skin = private_metadata.skin; // If hide_cancel is true, we don't need to extract information from the private_metadata if hide_cancel.unwrap_or(false) && action == "cancel" { @@ -463,7 +473,7 @@ async fn handle_submission( tracing::warn!("Failed to resolve slack token for {w_id}/{resource_path}: {e:#}"); Error::BadRequest("Invalid Slack callback request".to_string()) })?; - update_original_slack_message(action, slack_token, container).await?; + update_original_slack_message(action, slack_token, container, skin).await?; Ok(()) } @@ -475,14 +485,19 @@ async fn transform_schemas( required: Option>, default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, + skin: ApprovalSkin, ) -> Result { tracing::debug!("Resume urls: {:#?}", urls); + let link_label = match skin { + ApprovalSkin::Detailed => "Flow suspension details", + ApprovalSkin::Minimal => "View in Windmill", + }; let mut blocks = vec![serde_json::json!({ "type": "section", "text": { "type": "mrkdwn", - "text": format!("{}\n<{}|Flow suspension details>", text, urls.approvalPage), + "text": format!("{}\n<{}|{link_label}>", text, urls.approvalPage), } })]; @@ -918,10 +933,6 @@ async fn send_slack_message( value["approver"] = serde_json::json!(approver); } - if let Some(message) = message { - value["message"] = serde_json::json!(message); - } - if let Some(default_args_json) = default_args_json { value["default_args_json"] = default_args_json.clone(); } @@ -950,33 +961,8 @@ async fn send_slack_message( .map_err(|e| Box::new(e) as Box)?; value["signature"] = serde_json::json!(signature); - let payload = serde_json::json!({ - "channel": channel_id, - "text": "A flow has been suspended. Please approve or reject the flow.", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A flow has been suspended. Please approve or reject the flow." - } - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "View" - }, - "action_id": "open_modal", - "value": value.to_string() - } - ] - } - ] - }); + let skin = get_approval_step_skin(db, w_id, job_id, flow_step_id).await; + let payload = channel_message_payload(channel_id, skin, message, value); tracing::debug!("Payload: {:?}", payload); @@ -1000,6 +986,88 @@ async fn send_slack_message( Ok(StatusCode::OK) } +/// The channel post announcing the approval. Its button hands `button_value` to the modal, with +/// `message` added, shortened to what Slack's button value limit leaves room for. +fn channel_message_payload( + channel_id: &str, + skin: ApprovalSkin, + message: Option<&str>, + mut button_value: serde_json::Value, +) -> serde_json::Value { + let message = message.map(|m| message_fitting_button_value(&button_value, m)); + if let Some(message) = &message { + button_value["message"] = serde_json::json!(message); + } + let (text, section, button_label) = match skin { + ApprovalSkin::Detailed => { + let text = "A flow has been suspended. Please approve or reject the flow."; + (text, text.to_string(), "View") + } + ApprovalSkin::Minimal => { + let mut section = "*Approval requested*".to_string(); + if let Some(message) = &message { + section.push('\n'); + section.push_str(message); + } + ("Approval requested", section, "Review") + } + }; + + serde_json::json!({ + "channel": channel_id, + "text": text, + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": section + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": button_label + }, + "action_id": "open_modal", + "value": button_value.to_string() + } + ] + } + ] + }) +} + +/// The longest prefix of `message` that keeps `button_value` carrying it within Slack's limit. +fn message_fitting_button_value(button_value: &serde_json::Value, message: &str) -> String { + let mut with_message = button_value.clone(); + let mut fits = |max_chars: usize| { + let fitted = truncate_with_ellipsis(message, max_chars); + with_message["message"] = serde_json::json!(fitted); + (with_message.to_string().chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS).then_some(fitted) + }; + if let Some(whole) = fits(usize::MAX) { + return whole; + } + // Searched on the serialized length, which escaping makes longer than the raw prefix, and + // which grows with every character kept. + let (mut shortest, mut longest) = + (0, message.chars().count().min(SLACK_BUTTON_VALUE_MAX_CHARS)); + while shortest < longest { + let mid = (shortest + longest + 1) / 2; + if fits(mid).is_some() { + shortest = mid; + } else { + longest = mid - 1; + } + } + fits(shortest).unwrap_or_else(|| truncate_with_ellipsis(message, 0)) +} + async fn get_modal_blocks( db: DB, w_id: &str, @@ -1034,7 +1102,7 @@ async fn get_modal_blocks( ) .await?; - let ApprovalFormDetails { message_str, urls, schema } = approval_details; + let ApprovalFormDetails { message_str, urls, schema, skin } = approval_details; // Get the card content let card_content = transform_schemas( @@ -1063,6 +1131,7 @@ async fn get_modal_blocks( }), default_args_json, dynamic_enums_json, + skin, ) .await?; @@ -1077,6 +1146,7 @@ async fn get_modal_blocks( resume_button_text, cancel_button_text, &private_metadata_signature, + skin, ))) } @@ -1090,27 +1160,32 @@ fn construct_payload( resume_button_text: Option<&str>, cancel_button_text: Option<&str>, signature: &str, + skin: ApprovalSkin, ) -> serde_json::Value { + let (title, resume_label, cancel_label) = match skin { + ApprovalSkin::Detailed => ("Workflow Suspended", "Resume Workflow", "Cancel Workflow"), + ApprovalSkin::Minimal => ("Approval request", "Approve", "Reject"), + }; let mut view = serde_json::json!({ "type": "modal", "callback_id": "submit_form", "notify_on_close": true, "title": { "type": "plain_text", - "text": "Workflow Suspended" + "text": title }, "blocks": blocks, "submit": { "type": "plain_text", - "text": resume_button_text.unwrap_or("Resume Workflow") + "text": resume_button_text.unwrap_or(resume_label) }, - "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature }).to_string(), + "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature, "skin": skin }).to_string(), }); if !hide_cancel { view["close"] = serde_json::json!({ "type": "plain_text", - "text": cancel_button_text.unwrap_or("Cancel Workflow") + "text": cancel_button_text.unwrap_or(cancel_label) }); } @@ -1193,11 +1268,13 @@ async fn update_original_slack_message( action: &str, token: String, container: Container, + skin: ApprovalSkin, ) -> Result<(), Error> { - let message = if action == "resume" { - "\n\n*Workflow has been resumed!* :white_check_mark:" - } else { - "\n\n*Workflow has been canceled!* :x:" + let message = match (skin, action == "resume") { + (ApprovalSkin::Detailed, true) => "\n\n*Workflow has been resumed!* :white_check_mark:", + (ApprovalSkin::Detailed, false) => "\n\n*Workflow has been canceled!* :x:", + (ApprovalSkin::Minimal, true) => "*Approved* :white_check_mark:", + (ApprovalSkin::Minimal, false) => "*Rejected* :x:", }; let final_blocks = vec![serde_json::json!({ @@ -1242,3 +1319,66 @@ async fn update_original_slack_message( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_message_keeps_the_channel_post_within_slack_limits() { + let button_value = serde_json::json!({ + "w_id": "demo", + "job_id": Uuid::nil(), + "path": "u/admin/slack", + "channel": "C0123456789", + "flow_step_id": "a", + "signature": "f".repeat(64), + }); + let carried = |skin, message: &str| { + let payload = channel_message_payload("C1", skin, Some(message), button_value.clone()); + let button = payload["blocks"][1]["elements"][0]["value"] + .as_str() + .unwrap() + .to_string(); + assert!(button.chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS); + let section = payload["blocks"][0]["text"]["text"].as_str().unwrap(); + assert!(section.chars().count() <= 3000); + serde_json::from_str::(&button) + .unwrap() + .message + .unwrap() + }; + // Quotes and newlines each cost two characters once escaped into the button value. + let message = "Expense \"offsite\" line\n".repeat(1000); + for skin in [ApprovalSkin::Detailed, ApprovalSkin::Minimal] { + let kept = carried(skin, &message); + let kept = kept.strip_suffix("...").unwrap(); + assert!(message.starts_with(kept)); + assert!(kept.chars().count() > 1_000); + assert_eq!(carried(skin, "Short message"), "Short message"); + } + } + + #[test] + fn minimal_skin_survives_the_modal_round_trip() { + let container = Container { message_ts: "1".to_string(), channel_id: "C1".to_string() }; + let payload = construct_payload( + serde_json::json!([]), + false, + "trigger", + "https://example.com/resume", + "u/admin/slack", + container, + None, + None, + "signature", + ApprovalSkin::Minimal, + ); + let view = &payload["view"]; + assert_eq!(view["submit"]["text"], "Approve"); + assert_eq!(view["close"]["text"], "Reject"); + let metadata: PrivateMetadata = + serde_json::from_str(view["private_metadata"].as_str().unwrap()).unwrap(); + assert_eq!(metadata.skin, ApprovalSkin::Minimal); + } +} diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 06a79d022e..22645a2ac4 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -107,6 +107,14 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< .header("Cross-Origin-Resource-Policy", "cross-origin"); } + // Login and its siblings carry a different `rd` on every page that links + // to them, so a crawler meets thousands of URLs for one form. The app is + // client-rendered, so a meta tag only exists after a render pass; the + // header is seen on the first fetch. + if original_path.starts_with("/user/") { + res = res.header("X-Robots-Tag", "noindex, nofollow"); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index 2268a63894..a1a9206196 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -99,7 +99,6 @@ fn build_standard_scope_domains() -> Vec { ("configs", "Configs", "Configuration management", false), ("oauth", "OAuth", "OAuth management", false), ("ai", "AI", "AI feature management", false), - ("ai_skills", "AI Skills", "AI skill management", false), ( "ai_evals", "AI Evals", diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index e58aab44a2..387ba56589 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -1,6 +1,7 @@ use super::{ - http_trigger_args::RawHttpTriggerArgs, refresh_routers, AuthenticationMethod, HttpMethod, - RequestType, TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, + effective_allowed_origins, http_trigger_args::RawHttpTriggerArgs, match_origin, + refresh_routers, AuthenticationMethod, HttpMethod, RequestType, TriggerRoute, + HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, }; use crate::{ auth::{AuthCache, OptTokened}, @@ -24,6 +25,7 @@ use std::{collections::HashMap, sync::Arc}; use windmill_common::{ db::UserDB, error::{Error, Result}, + global_settings::HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS, jobs::JobTriggerKind, triggers::{TriggerKind, TriggerMetadata}, utils::{not_found_if_none, StripPath}, @@ -37,12 +39,222 @@ use { windmill_object_store::build_object_store_client, }; +/// Which router a request's CORS decision must be looked up in. +/// +/// A preflight names the method it is asking about in +/// `Access-Control-Request-Method`; the routers are keyed by method, so without +/// that header there is nothing to look up. +fn cors_lookup_method(req: &axum::extract::Request) -> Option { + let method = req.method(); + if method == http::Method::OPTIONS { + req.headers() + .get(http::header::ACCESS_CONTROL_REQUEST_METHOD) + .and_then(|method| method.to_str().ok()) + .and_then(|method| http::Method::try_from(method).ok()) + .as_ref() + .and_then(routable_method) + } else { + routable_method(method) + } +} + +/// The router key a request method maps to. `HEAD` resolves the `GET` route it +/// mirrors, and does so for a preflight naming it too: browsers send +/// `Access-Control-Request-Method: HEAD` when the HEAD carries a non-safelisted +/// header, and answering that preflight from a different route than the request +/// itself resolves is how the two come to disagree. +fn routable_method(method: &http::Method) -> Option { + if method == http::Method::HEAD { + Some(HttpMethod::Get) + } else { + HttpMethod::try_from(method).ok() + } +} + +/// The key to look a request up by, matching what `route_job` resolves it to. +/// +/// `Path` percent-decodes before `get_http_route_trigger` builds its +/// lookup key, so decoding here is what keeps the two agreeing: on the raw path, +/// `/us%65rs` misses the trigger registered at `/users` that goes on to serve the +/// request, and the response would carry the permissive default instead of that +/// trigger's allowlist. +fn cors_lookup_path(raw_path: &str) -> Option { + let decoded = urlencoding::decode(raw_path).ok()?; + // `StripPath::to_path` strips one leading slash and the handler trims + // trailing ones, before a single `/` is prefixed back on. + let stripped = decoded.strip_prefix('/').unwrap_or(&decoded); + Some(format!("/{}", stripped.trim_end_matches('/'))) +} + +/// What the middleware should stamp, decided while the routers guard is held. +/// +/// Deliberately small and owned: the allowlist itself never leaves the guard, +/// so a large one is scanned in place instead of being copied per request onto +/// a path an unauthenticated preflight can reach. +#[derive(Clone)] +enum CorsDecision { + /// No allowlist applies, so the permissive default stands. + Unrestricted, + /// An allowlist applies. `allow_origin` is the value to echo, present only + /// when the request's own `Origin` is on the list. + Restricted { route_method: Option, allow_origin: Option }, + /// The routers could not be read, so nothing is known about this path. + Unavailable, +} + +/// Whether a route actually serves a static website, rather than merely saying +/// it does. +/// +/// `is_static_website` is a caller-set flag that validation ties to nothing: a +/// route can carry it while having no assets configured and a `script_path` +/// that `route_job` runs regardless. Keying the exemption off the flag alone +/// would let one boolean disable a route's allowlist and hand its runnable back +/// the `wm_headers` escape hatch, so the assets have to be there too. +fn serves_a_static_website(trigger: &TriggerRoute) -> bool { + trigger.is_static_website && trigger.static_asset_config.is_some() +} + +/// A static website is never subject to an allowlist, its own included. It has +/// no authentication of its own — the editor does not offer any — so it hands +/// out public files that any non-browser client can already fetch, and +/// restricting which browsers may read them protects nothing while breaking the +/// cross-origin uses that do consult CORS: a webfont, a `crossorigin` asset, a +/// `fetch`. +/// +/// A single-file static asset is not exempt. That one can carry an +/// `authentication_method`, so its content need not be public, and an allowlist +/// is what keeps another origin from reading a response its own credentials +/// would not have obtained. +/// +/// The CORS verdict for a request, published by whoever resolved its trigger. +/// +/// The middleware stamps headers after the handler returns, but only the +/// handler knows which trigger it actually served. Re-deriving that from the +/// routers cache is a second lookup which can disagree with the first when a +/// route is edited, deleted or widened mid-request, and every ordering of the +/// two is wrong in some case. So the verdict travels with the request instead +/// of being worked out twice. +#[derive(Clone, Default)] +struct ResolvedCorsPolicy(std::sync::Arc>); + +impl ResolvedCorsPolicy { + /// Record what the trigger being served allows. Called once, where the + /// route is resolved, so the answer cannot drift from the response. + fn publish(&self, trigger: &TriggerRoute, method: Option, headers: &HeaderMap) { + let decision = if serves_a_static_website(trigger) { + CorsDecision::Unrestricted + } else { + let instance_default = HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.load(); + match effective_allowed_origins( + trigger.allowed_origins.as_deref(), + instance_default.as_slice(), + ) { + None => CorsDecision::Unrestricted, + Some(allowed_origins) => CorsDecision::Restricted { + route_method: method, + allow_origin: match_origin(allowed_origins, headers.get(http::header::ORIGIN)), + }, + } + }; + let _ = self.0.set(decision); + } + + fn published(&self) -> Option { + self.0.get().cloned() + } +} + +/// Decide the CORS answer from the routers cache, for a request no handler +/// published a verdict for: a preflight, an unknown path, or any failure ahead +/// of the publish — authentication included, which runs after the route itself +/// resolves. +/// +/// Loads the routers when the cache is cold, the way `get_http_route_trigger` +/// does, so a preflight is answered from the same view of the routes as the +/// request that follows it. +async fn resolve_cors_decision( + db: &DB, + http_method: HttpMethod, + requested_path: &str, + origin: Option<&http::HeaderValue>, +) -> CorsDecision { + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + + let routers_cache = if routers_cache.routers.is_empty() { + drop(routers_cache); + match refresh_routers(db, false).await { + Ok((_, routers_cache)) => routers_cache, + Err(err) => { + tracing::error!("Could not load HTTP routers to resolve CORS: {err:#}"); + return CorsDecision::Unavailable; + } + } + } else { + routers_cache + }; + + let Some(router) = routers_cache.routers.get(&http_method) else { + return CorsDecision::Unavailable; + }; + + let route = router.at(requested_path).ok(); + if route + .as_ref() + .is_some_and(|trigger| serves_a_static_website(trigger.value)) + { + return CorsDecision::Unrestricted; + } + let route_allowed_origins = route + .as_ref() + .and_then(|trigger| trigger.value.allowed_origins.as_deref()); + + let instance_default = HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS.load(); + match effective_allowed_origins(route_allowed_origins, instance_default.as_slice()) { + None => CorsDecision::Unrestricted, + Some(allowed_origins) => CorsDecision::Restricted { + route_method: route.map(|_| http_method), + allow_origin: match_origin(allowed_origins, origin), + }, + } +} + async fn conditional_cors_middleware( - req: axum::extract::Request, + Extension(db): Extension, + mut req: axum::extract::Request, next: axum::middleware::Next, ) -> Response { + let origin = req.headers().get(http::header::ORIGIN).cloned(); + // Owned before `next.run` consumes the request. `&Request` is not `Send` + // (`Body` is not `Sync`), so nothing borrowed from it can cross the await. + // The URI is carried rather than the decoded path: cloning it is a refcount + // bump, while decoding allocates, and only the fallback below ever needs it. + let lookup_method = cors_lookup_method(&req); + let uri = req.uri().clone(); + + let resolved = ResolvedCorsPolicy::default(); + req.extensions_mut().insert(resolved.clone()); + let mut response = next.run(req).await; + let decision = match resolved.published() { + // The handler resolved a trigger and said what it served under. That is + // the policy this response was produced with, so nothing else can be + // more authoritative. + Some(decision) => decision, + // No verdict was published: a preflight, an unknown path, or a request + // that failed before reaching the publish, authentication included. No + // runnable produced this body, so reading the cache cannot contradict + // anything. + None => match lookup_method.zip(cors_lookup_path(uri.path())) { + Some((method, path)) => { + resolve_cors_decision(&db, method, &path, origin.as_ref()).await + } + // Not a preflight, not a routable method, or a path that does not + // decode. + None => CorsDecision::Unrestricted, + }, + }; + let headers = response.headers_mut(); // Check existing headers first to determine what not to insert @@ -67,18 +279,65 @@ async fn conditional_cors_middleware( } } - // Insert only the missing headers - if !not_insert_origin { - headers.insert( - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, - http::HeaderValue::from_static("*"), - ); + match &decision { + CorsDecision::Restricted { allow_origin, .. } => { + // A configured allowlist decides, overriding any `wm_headers` value + // the runnable set. The preflight is answered before any code runs, + // so config is the only thing it can consult; letting the response + // widen what the preflight advertised would make the two disagree + // and leave the allowlist bounding nothing. A route escapes a + // stricter instance default — `wm_headers` included — by setting + // its own list to `*`. + match allow_origin { + Some(value) => { + headers.insert(http::header::ACCESS_CONTROL_ALLOW_ORIGIN, value.clone()) + } + // No match: omit the header entirely so the browser blocks the + // read, and drop any value the runnable set. + None => headers.remove(http::header::ACCESS_CONTROL_ALLOW_ORIGIN), + }; + // Appended, not inserted: the answer now depends on the request's + // Origin, and a shared cache that ignores it would hand one + // origin's response to another. + headers.append(http::header::VARY, http::HeaderValue::from_static("origin")); + } + // The routers could not be read, so nothing is known about this path; + // only a preflight or an unresolved request reaches here. Answering a + // preflight permissively would let a disallowed origin go on to invoke + // a runnable whose purpose may be a side effect. + CorsDecision::Unavailable => { + headers.remove(http::header::ACCESS_CONTROL_ALLOW_ORIGIN); + } + CorsDecision::Unrestricted => { + if !not_insert_origin { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, + http::HeaderValue::from_static("*"), + ); + } + } } if !not_insert_methods { + // A route accepts exactly one method, so advertising all seven + // overstates it. Only a route under an allowlist gets the narrower + // answer; an unrestricted one advertises the full supported set, since + // narrowing it would say something about a route the response is not + // otherwise willing to disclose. + let restricted_method = match &decision { + CorsDecision::Restricted { route_method, .. } => *route_method, + _ => None, + }; headers.insert( http::header::ACCESS_CONTROL_ALLOW_METHODS, - http::HeaderValue::from_static("GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"), + http::HeaderValue::from_static(match restricted_method { + Some(HttpMethod::Get) => "GET, OPTIONS", + Some(HttpMethod::Post) => "POST, OPTIONS", + Some(HttpMethod::Put) => "PUT, OPTIONS", + Some(HttpMethod::Delete) => "DELETE, OPTIONS", + Some(HttpMethod::Patch) => "PATCH, OPTIONS", + None => "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS", + }), ); } @@ -237,6 +496,7 @@ async fn route_job( Extension(db): Extension, Extension(user_db): Extension, Extension(auth_cache): Extension>, + Extension(cors_policy): Extension, OptTokened { token }: OptTokened, Path(route_path): Path, headers: HeaderMap, @@ -255,6 +515,10 @@ async fn route_job( .await .map_err(|e| e.into_response())?; + // Publish before anything else can fail: the CORS middleware stamps this + // response either way, and it must reflect the trigger actually served. + cors_policy.publish(&trigger, routable_method(&args.0.metadata.method), &headers); + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { return Err(Error::NotFound(format!( "Runnable path of HTTP route at path: {}", diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 8804004f2b..a7f9aeda89 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -55,6 +55,7 @@ pub fn global_service() -> Router { .route("/rename/{user}", post(rename_user)) .route("/onboarding", post(submit_onboarding_data)) .route("/ext_jwt_tokens", get(list_ext_jwt_tokens)) + .route("/guests", get(list_guests)) .route( "/offboard_preview/{user}", get(crate::offboarding::global_offboard_preview), @@ -141,6 +142,58 @@ async fn list_ext_jwt_tokens( Ok(Json(rows)) } +#[derive(serde::Serialize, sqlx::FromRow)] +pub struct GuestActivity { + pub email: String, + pub workspaces: Vec, + pub first_seen: chrono::NaiveDate, + pub last_seen: chrono::NaiveDate, +} + +#[derive(serde::Serialize)] +pub struct GuestList { + pub usage: windmill_common::workspaces::GuestUsage, + pub guests: Vec, +} + +#[derive(serde::Deserialize)] +struct ListGuestsQuery { + page: Option, + per_page: Option, +} + +/// The distinct guests of the trailing window, the set the allowance is counted on, +/// most recently seen first. +async fn list_guests( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> Result> { + require_super_admin(&db, &authed).await?; + + let (per_page, offset) = windmill_common::utils::paginate(windmill_common::utils::Pagination { + page: query.page, + per_page: query.per_page, + }); + let usage = windmill_common::workspaces::guest_usage(&db).await?; + let guests = sqlx::query_as::<_, GuestActivity>( + "SELECT email, array_agg(DISTINCT workspace_id) AS workspaces, + MIN(day) AS first_seen, MAX(day) AS last_seen + FROM guest_activity + WHERE day > CURRENT_DATE - $3 + GROUP BY email + ORDER BY MAX(day) DESC, email + LIMIT $1 OFFSET $2", + ) + .bind(per_page as i64) + .bind(offset as i64) + .bind(windmill_common::workspaces::GUEST_WINDOW_DAYS) + .fetch_all(&db) + .await?; + + Ok(Json(GuestList { usage, guests })) +} + async fn set_password( Extension(db): Extension, Extension(argon2): Extension>>, @@ -208,6 +261,12 @@ async fn rename_user( ))); } + let old_instance_username = + sqlx::query_scalar!("SELECT username FROM password WHERE email = $1", user_email) + .fetch_optional(&mut *tx) + .await? + .flatten(); + sqlx::query!( "UPDATE password SET username = $1 WHERE email = $2", ru.new_username, @@ -216,6 +275,36 @@ async fn rename_user( .execute(&mut *tx) .await?; + // The per-workspace sweep below only reaches accounts with a `usr` row. A superadmin acting + // outside their workspaces has none, yet an app can name them: their principal is + // `u/{password.username}`, which this rename just moved. Matching on the address as well + // keeps a like-named member of some other workspace out of it. + if let Some(old_username) = old_instance_username.filter(|u| *u != ru.new_username) { + let old_principal = windmill_common::users::username_to_permissioned_as(&old_username); + let new_principal = + windmill_common::users::username_to_permissioned_as(&ru.new_username); + sqlx::query!( + "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb($1::text)) + WHERE policy->>'on_behalf_of' = $2 AND policy->>'on_behalf_of_email' = $3", + &new_principal, + &old_principal, + user_email + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb($1::text))) + WHERE typ IN ('app', 'raw_app') + AND value->'policy'->>'on_behalf_of' = $2 + AND value->'policy'->>'on_behalf_of_email' = $3"#, + &new_principal, + &old_principal, + user_email + ) + .execute(&mut *tx) + .await?; + } + let workspace_usernames = sqlx::query!( "SELECT workspace_id, username FROM usr WHERE email = $1", &user_email @@ -263,6 +352,17 @@ async fn update_username_in_workpsace<'c>( new_username: &str, w_id: &str, ) -> error::Result<()> { + // ---- data table tenants ---- + // Tenants name the user, so the rename has to follow here too; a list left naming the old + // username silently drops the access instead of moving it. + windmill_common::workspaces::rename_datatable_tenant_in_workspace( + tx, + w_id, + &format!("u/{old_username}"), + &format!("u/{new_username}"), + ) + .await?; + // ---- instance and workspace users ---- sqlx::query!( "UPDATE usr SET username = $1 WHERE email = $2", @@ -701,6 +801,17 @@ async fn update_username_in_workpsace<'c>( .execute(&mut **tx) .await?; + // An app draft carries a copy of the deployed policy, so the rename must reach it + // there too — same reason as the script/flow draft sweep above. + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['policy', 'on_behalf_of'], to_jsonb('u/' || $1))) WHERE typ IN ('app', 'raw_app') AND value->'policy'->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( "UPDATE app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", new_username, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 01eb32b7f6..186cfbc291 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,7 +18,6 @@ use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; - use axum::{ extract::{Extension, Path}, routing::{get, post}, @@ -109,6 +108,8 @@ async fn edit_copilot_config( } ai_config.validate_model_pricing()?; + ai_config.validate_token_limits()?; + ai_config.validate_sessions_retention()?; let mut tx = db.begin().await?; @@ -147,19 +148,38 @@ async fn edit_copilot_config( .await?; let workspace_has_config = ai_config.has_providers(); + let copilot_disabled = ai_config.copilot_disabled; + let sessions_storage_disabled = ai_config.sessions_storage_disabled; + let sessions_retention_days = ai_config.sessions_retention_days; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await?; let settings_state = build_copilot_settings_state(workspace_has_config, instance_ai_config.as_ref()); - let effective_ai_config = if workspace_has_config { + // A provider-less instance config (e.g. `{}`) is unconfigured, same as build_copilot_settings_state + // treats it — so it must not shadow the free-tier fallback here either. + let instance_config_with_providers = instance_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .filter(|c| c.has_providers()); + let mut effective_ai_config = if workspace_has_config { ai_config - } else if let Some(instance_ai_config) = instance_ai_config { - serde_json::from_value::(instance_ai_config).unwrap_or_default() + } else if let Some(instance_config) = instance_config_with_providers { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Same fallback as get_copilot_info: with nothing configured, surface Windmill's free + // tier (EE-only) so clearing a workspace provider activates it immediately, instead of + // returning an empty config that disables AI until the next page reload re-fetches it. + free_config } else { AIConfig::default() }; + effective_ai_config.copilot_disabled = copilot_disabled; + effective_ai_config.sessions_storage_disabled = sessions_storage_disabled; + effective_ai_config.sessions_retention_days = sessions_retention_days; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -179,6 +199,7 @@ struct EditCopilotConfigResponse { } async fn get_copilot_info( + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { @@ -194,19 +215,42 @@ async fn get_copilot_info( )) })?; - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = + let copilot_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.copilot_disabled); + let sessions_storage_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.sessions_storage_disabled); + let sessions_retention_days = workspace_ai_config + .as_ref() + .and_then(|c| c.0.sessions_retention_days); + let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await? - { - Ok(Json( - serde_json::from_value::(instance_config).unwrap_or_default(), - )) - } else { - Ok(Json(AIConfig::default())) - } + .and_then(|v| serde_json::from_value::(v).ok()) + // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the + // free-tier fallback, matching the proxy and edit_copilot_config paths. + .filter(|c| c.has_providers()); + let mut effective = + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + workspace_ai_config.0 + } else if let Some(instance_config) = instance_config { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + free_config + } else { + AIConfig::default() + }; + effective.copilot_disabled = copilot_disabled; + effective.sessions_storage_disabled = sessions_storage_disabled; + effective.sessions_retention_days = sessions_retention_days; + Ok(Json(effective)) } #[cfg(feature = "enterprise")] @@ -216,7 +260,14 @@ pub async fn get_critical_alerts( authed: ApiAuthed, Query(params): Query, ) -> JsonResult { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::get_critical_alerts(db, params, Some(w_id)).await } @@ -232,7 +283,14 @@ pub async fn acknowledge_critical_alert( Path((w_id, id)): Path<(String, i32)>, authed: ApiAuthed, ) -> Result { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 70910c885e..39bcb13937 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -149,9 +149,7 @@ async fn derive_email( if let Some(hit) = cache.get(permissioned_as) { return Ok(Some(hit.clone())); } - // Uncached: the address goes into an archive a client redeploys from, and the write path - // validates the pair it sends back against an uncached lookup. The memo above still holds - // this to one query per distinct principal per export. + // The memo above holds this to one query per distinct principal per export. let email = windmill_common::users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db) .await?; @@ -346,7 +344,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +363,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -667,7 +665,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -928,7 +926,7 @@ pub(crate) async fn tarball_workspace( if !skip_resource_types.unwrap_or(false) { let resource_types = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", &w_id ) .fetch_all(&mut *tx) @@ -1004,8 +1002,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; @@ -1437,9 +1434,11 @@ pub(crate) async fn tarball_workspace( // Native triggers (Nextcloud, Google Drive, GitHub) are never // cloned into a fork — a fork only has one if its owner created // it there, so it's always "fork-only" and keeps its own mode. - // No parent-value substitution applies; we only strip the - // webhook token hash. - let native_ignore_keys = vec!["webhook_token_hash"]; + // No parent-value substitution applies; we strip the webhook + // token hash, and `enabled`, which is operational state a sync + // deliberately does not carry — whether a trigger is paused + // belongs to the workspace it runs in, not to the code. + let native_ignore_keys = vec!["webhook_token_hash", "enabled"]; for trigger in native_triggers { let trigger_str = &to_string_without_metadata( @@ -1590,10 +1589,11 @@ pub(crate) async fn tarball_workspace( .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility - // Server-owned auto-pull state (the HMAC webhook secret + hook id/error and - // the synced-sha / last-pull status) must never leave the server: keep it out - // of export archives and synced repos, and don't let a re-imported workspace - // inherit another install's hook/sync state. Mirrors the GET-settings redaction. + // Server-owned state (the HMAC webhook secret + hook id/error, the + // synced-sha / last-pull status, the admin automatic pulls run as, and what + // the credential check observed) must never leave the server: keep it out of + // export archives and synced repos, and don't let a re-imported workspace + // inherit another install's hook/sync state or pull identity. fn redact_git_sync_for_export(git_sync: Option) -> Option { let mut git_sync = git_sync?; if let Some(repos) = git_sync @@ -1610,10 +1610,18 @@ pub(crate) async fn tarball_workspace( "webhook_error", "last_synced_sha", "last_pull_status", + "enabled_by", ] { auto_pull.remove(field); } } + // What this install observed about its own credential: a token + // id and expiry, and a `checked_at` that moves on its own. + // None of it describes the workspace, and in a git-synced + // `wmill.yaml` it would churn the file for no reason. + if let Some(repo) = repo.as_object_mut() { + repo.remove("credential"); + } } } Some(git_sync) @@ -1635,7 +1643,7 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color.clone(), operator_settings: row.operator_settings.clone(), - datatable: row.datatable.clone(), + datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable.clone()), slack_team_id: row.slack_team_id.clone(), slack_name: row.slack_name.clone(), slack_command_script: row.slack_command_script.clone(), @@ -1699,7 +1707,7 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color, operator_settings: row.operator_settings, - datatable: row.datatable, + datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable), slack_team_id: row.slack_team_id, slack_name: row.slack_name, slack_command_script: row.slack_command_script, diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index e2c3233183..06de7cc4aa 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -33,6 +33,7 @@ path = "src/lib.rs" tar.workspace = true hmac.workspace = true sha2.workspace = true +sha1.workspace = true thiserror.workspace = true anyhow.workspace = true serde.workspace = true @@ -109,6 +110,8 @@ pep440_rs.workspace = true systemstat.workspace = true size.workspace = true rsa = { workspace = true, optional = true } +spki = { workspace = true } +pkcs1 = { workspace = true } aes-gcm = { workspace = true, optional = true } semver.workspace = true diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt:// → `. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index fa1bc96334..c12d62f206 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -19,10 +19,11 @@ use crate::{ }; /// Whether `label` denotes a user-created token rather than a system token -/// (`session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`). System-token -/// labels are load-bearing — session cleanup, super_admin propagation, expiry -/// notifications and username overrides all key off them — so they must not be -/// user-editable. `None` (no label) is treated as a user token. +/// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`, +/// `embed_app:*`, `sdk_app:*`, `impersonation:*`, `cli-login:*`). System-token labels are load-bearing — +/// session cleanup, super_admin propagation, expiry notifications and username overrides +/// all key off them — so they must not be user-editable. `None` (no label) is treated as +/// a user token. /// /// This is the canonical copy. When updating it, also update its mirrors: /// - the `update_token_label` editability guard (SQL `WHERE`) in @@ -36,18 +37,48 @@ pub fn is_user_token(label: Option<&str>) -> bool { // frontend mirror (`label.toLowerCase().startsWith('ephemeral')`) and // the SQL `lower(label) NOT LIKE 'ephemeral%'` guard. l != "session" + && l != GUEST_SESSION_LABEL && !l.to_lowercase().starts_with("ephemeral") && l != "debugger-token" && !l.starts_with("mcp-oauth-") + // Short-lived tokens the server mints per app open or per service-account + // impersonation (EE `users_ee.rs`) and nobody manages, so an expiry warning + // for one is noise. + && !l.starts_with(APP_EMBED_TOKEN_LABEL_PREFIX) + && !l.starts_with(RAW_APP_SDK_TOKEN_LABEL_PREFIX) + && !l.starts_with("impersonation:") + && !l.starts_with(CLI_LOGIN_TOKEN_LABEL_PREFIX) } } } +/// How far ahead of a user token's expiration its owner is warned (`check_expiring_tokens` in +/// the monitor). A token whose whole lifetime fits in this window gets no warning at all: it +/// would arrive minutes after creation, about a lifetime its creator just picked. Its +/// "expired and deleted" notice still goes out. +pub const TOKEN_EXPIRY_WARNING_DAYS: i32 = 7; + +/// Label prefix, followed by the app path, of the token an app viewer's sandboxed iframe +/// runs with. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out. +pub const APP_EMBED_TOKEN_LABEL_PREFIX: &str = "embed_app:"; + +/// Label prefix, followed by the app path, of the token a raw app's bundle uses for the +/// frontend SDK. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out. +pub const RAW_APP_SDK_TOKEN_LABEL_PREFIX: &str = "sdk_app:"; + +/// Label prefix, followed by the username, of the token the CLI authorization page mints for +/// `wmill` logins. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out: +/// the CLI signs in again on its own once that token expires, so an expiry email for it asks +/// the user to do nothing. Not in [`is_server_minted_label`], since the page mints it through +/// `/users/tokens/create`. +pub const CLI_LOGIN_TOKEN_LABEL_PREFIX: &str = "cli-login:"; + /// Whether `label` belongs to a namespace only the server mints, and which therefore must be /// rejected by `create_token`. Narrower than [`is_user_token`], which also drives label -/// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token` -/// and `debugger-token` are minted by the editor and the debugger through that same handler, -/// so reserving them would break those features. +/// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token`, +/// `debugger-token` and `ephemeral-test-connection: *` are minted by the editor, the debugger +/// and object-storage connection tests through that same handler, so reserving them would +/// break those features. /// /// `username_override_from_label` trusts a label to name the entity acting only if it is in /// here, so anything added must be unmintable by a member. @@ -56,9 +87,40 @@ pub fn is_server_minted_label(label: &str) -> bool { || label.starts_with("ephemeral-script-end-user-") || label == "ephemeral-script" || label == "session" + || label == GUEST_SESSION_LABEL || label.starts_with("mcp-oauth-") } +/// Label on a guest session (the `guest` app execution mode). This is the *grant*: +/// `AuthCache` will resolve a token carrying it into an identity with no account behind +/// it, which nothing else can do. It must therefore stay unforgeable, which is what +/// listing it in [`is_server_minted_label`] buys — `/users/tokens/create` refuses it. +/// +/// Do not move this test onto the token's scopes. Scopes on a user-minted token are +/// caller-supplied and only ever *narrow* (`app_embed`, `raw_app_sdk`), so a scope +/// that granted non-member access would be free for anyone to declare. +pub const GUEST_SESSION_LABEL: &str = "guest_session"; + +/// Whether `label` marks a guest session. See [`GUEST_SESSION_LABEL`]. +/// +/// Reserved in [`is_user_token`] as well as [`is_server_minted_label`]: the former +/// gates relabelling, and a user token that could be relabelled *into* this +/// namespace would become a guest session with no workspace pin — one that +/// authenticates everywhere. +pub fn is_guest_session_label(label: Option<&str>) -> bool { + label == Some(GUEST_SESSION_LABEL) +} + +/// Whether `path` can be spliced into a scope as one literal resource. The scope +/// grammar reserves three characters: `:` separates the parts, `,` separates +/// resources, `*` is a wildcard. App paths are otherwise free-form (spaces, `@`). A +/// leading `/` is refused too: routes strip it, so the scope would never match. +pub fn is_scope_literal_path(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && !path.chars().any(|c| matches!(c, ':' | ',' | '*')) +} + /// Whether `label` is the one minted for a browser session at login. [`is_server_minted_label`] /// stops a member minting it directly, but `/users/refresh_token` hands one to any authenticated /// caller, so this attributes a request to the UI without proving it: never gate authority on it. @@ -420,6 +482,44 @@ async fn fetch_authed_from_permissioned_as_inner( w_id: &str, conn: &mut sqlx::PgConnection, ) -> Result { + // The `usr` row is the live binding between a `u/` principal and an address, and it is read + // here anyway for the workspace role. Callers may hand us a cached address, so read it before + // anything is granted: `super_admin` and `email_to_igroup` below are keyed on the address + // while the role is keyed on the principal, and an address that no longer belongs to this + // principal — a username freed and reassigned while its previous holder keeps a privileged + // account — would mix one account's role with another's instance privileges. + let member = match permissioned_as.split_once('/') { + Some(("u", name)) => { + sqlx::query!( + "SELECT is_admin, operator, email FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id + ) + .fetch_optional(&mut *conn) + .await? + } + _ => None, + }; + let resolved_email; + let email = match member.as_ref() { + Some(m) => m.email.as_str(), + // No enabled `usr` row. Resolve as `resolve_username_to_email` does: a disabled member's + // own row still wins over the `password` superadmin fallback, so it can never resolve to + // an unrelated superadmin who shares the username (workspace usernames are only unique per + // workspace). Off the member path, which is why it is worth a query that path skips. + None => match permissioned_as.split_once('/') { + Some(("u", name)) => { + resolved_email = + crate::users::resolve_username_to_email(w_id, name, &mut *conn).await?; + // No live binding at all: the supplied address stands. A cached one is at most one + // notify poll stale; accepted, see `users::get_email_from_permissioned_as`. + resolved_email.as_deref().unwrap_or(email) + } + _ => email, + }, + }; + let is_super_admin = permissioned_as == SUPERADMIN_SYNC_EMAIL || email == SUPERADMIN_SECRET_EMAIL || email == SUPERADMIN_NOTIFICATION_EMAIL @@ -433,22 +533,12 @@ async fn fetch_authed_from_permissioned_as_inner( if prefix == "u" { let (is_admin, is_operator) = if is_super_admin { (true, false) + } else if let Some(m) = member.as_ref() { + (m.is_admin, m.operator) } else { - let r = sqlx::query!( - "SELECT is_admin, operator FROM usr where username = $1 AND \ - workspace_id = $2 AND disabled = false", - name, - &w_id - ) - .fetch_optional(&mut *conn) - .await?; - if let Some(r) = r { - (r.is_admin, r.operator) - } else { - return Err(Error::NotFound(format!( - "user {name} not found in workspace {w_id}" - ))); - } + return Err(Error::NotFound(format!( + "user {name} not found in workspace {w_id}" + ))); }; let groups = get_groups_for_user(w_id, &name, email, &mut *conn).await?; @@ -903,6 +993,10 @@ mod tests { assert!(!is_user_token(Some("Ephemeral lsp token"))); assert!(!is_user_token(Some("debugger-token"))); assert!(!is_user_token(Some("mcp-oauth-client"))); + assert!(!is_user_token(Some("embed_app:f/team/dashboard"))); + assert!(!is_user_token(Some("sdk_app:u/admin/raw app"))); + assert!(!is_user_token(Some("impersonation:admin@windmill.dev"))); + assert!(!is_user_token(Some("cli-login:admin"))); } #[test] diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs new file mode 100644 index 0000000000..a9195055ac --- /dev/null +++ b/backend/windmill-common/src/datatable_roles.rs @@ -0,0 +1,362 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The instance's data table role catalog. +//! +//! A data table role is a real Postgres login role on the Windmill cluster, named exactly as the +//! user named it, shared by every instance database. Windmill decides who may ask for a role (the +//! per-data-table tenant lists in [`crate::workspaces`]); Postgres decides what the role may then +//! touch. The catalog here is only the first half's vocabulary plus the cluster provisioning. +//! +//! Entries are keyed by a generated id so a rename moves nothing else: tenants name the id. + +use std::collections::BTreeMap; + +use crate::{ + error::{Error, Result}, + DB, +}; + +/// The connection every data table resolved to before roles existed (`custom_instance_user`). It +/// owns every pre-existing object, so it is a reserved name rather than a catalog entry: never +/// created, renamed or dropped. +pub const ADMIN_DATATABLE_ROLE: &str = "admin"; + +/// The login the admin connection uses, and the role every created role is granted to — that +/// membership is what later lets it `ALTER ... OWNER TO` a role and drop it. +pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user"; + +/// One catalog entry, as stored in `datatable_role`. The password is per role and instance-wide; +/// it belongs to the instance, not to any workspace's settings. +/// No `Serialize`/`Deserialize`: the catalog is rows now, and a derived `Serialize` would emit +/// `pwd` — the same way out for a credential that the hand-written `Debug` below closes on the log +/// side. +#[derive(Clone)] +pub struct InstanceDatatableRole { + /// The Postgres role name, verbatim. + pub name: String, + pub enabled: bool, + /// Absent only for a role whose provisioning did not finish; resolving as it then errors + /// rather than falling back to admin. + /// + /// A plain string rather than a `StringOrSecretRef` like the instance user's password: that + /// one is a secret ref because an operator supplies it and may want it to come from their own + /// backend, while this one is minted here and never entered by anyone, so there is nothing for + /// a ref to point at. Encrypting generated secrets at rest is a separate change that would + /// take the replication password with it. + pub pwd: Option, +} + +/// Hand-written so `{:?}` on a catalog cannot put a live Postgres password in a log line or an +/// audit record. Everything else about the entry is safe to print. +impl std::fmt::Debug for InstanceDatatableRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InstanceDatatableRole") + .field("name", &self.name) + .field("enabled", &self.enabled) + .field("pwd", &self.pwd.as_ref().map(|_| "")) + .finish() + } +} + +pub type DatatableRoleCatalog = BTreeMap; + +/// Names Postgres or Windmill already owns. `admin` is excluded because it never reaches the +/// cluster as a role name at all — it resolves to `custom_instance_user`. +fn is_reserved_role_name(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + lower == ADMIN_DATATABLE_ROLE + || lower == "postgres" + || lower == "public" + || lower.starts_with("pg_") + || lower.starts_with("windmill_") + || lower.starts_with("custom_instance_") +} + +/// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a +/// quoted identifier, a `-- role ` annotation, and a `?role=` query parameter. +pub fn validate_role_name(name: &str) -> Result<()> { + if name.is_empty() || name.len() > 63 { + return Err(Error::BadRequest(format!( + "Invalid data table role name '{name}': it must be between 1 and 63 characters" + ))); + } + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(Error::BadRequest(format!( + "Invalid data table role name '{name}': only letters, digits, '_' and '-' are allowed" + ))); + } + if is_reserved_role_name(name) { + return Err(Error::BadRequest(format!( + "'{name}' is reserved and cannot be used as a data table role name" + ))); + } + Ok(()) +} + +/// A double-quoted Postgres identifier. Doubling `"` is Postgres's own escaping inside one, so this +/// quotes any name — schema, table or role. Role names are validated as well +/// ([`validate_role_name`]) because they also travel unquoted, in `-- role ` and `?role=`. +pub fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// Serialize the mutations that are not already serialized by the row itself. +/// +/// A create is an insert and a delete is a delete, which Postgres orders for us — the unique index +/// on `name` is what makes two concurrent creates of the same name one winner and one error. What +/// still needs it is the window between the cluster DDL and the row: `CREATE ROLE` is not visible +/// to another transaction's `pg_roles` check until commit, so without this two creates of the same +/// name both pass their existence check and one fails on the index having already made the login. +/// Held for the transaction, so the DDL has to run on that same transaction to be covered. +pub async fn lock_role_catalog(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<()> { + sqlx::query!("SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))") + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// A replication stream reads every row whatever a data table's roles grant. Turning roles on looks +/// for streams holding this exclusive; whatever can start a Postgres trigger or capture streaming +/// holds it shared on the transaction that commits it. So either the look sees the stream, or the +/// stream's listener connects after roles are committed and refuses. Held for the transaction. +pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bool) -> Result<()> { + let lock = if exclusive { + "pg_advisory_xact_lock" + } else { + "pg_advisory_xact_lock_shared" + }; + sqlx::query(&format!("SELECT {lock}(hashtext('datatable_streams'))")) + .execute(conn) + .await?; + Ok(()) +} + +/// Whether an instance database is reached only through entries under roles is decided by two +/// writes that lock different workspaces' settings rows: turning roles on for one entry, and a +/// settings save pointing an entry without roles at the database. Each holds this for every +/// database it decides on, so neither reads past the other's uncommitted write. Held for the +/// transaction; the names are locked in sorted order so two holders cannot deadlock. +pub async fn lock_instance_databases_governance<'a>( + conn: &mut sqlx::PgConnection, + dbnames: impl IntoIterator, +) -> Result<()> { + let dbnames: std::collections::BTreeSet<&str> = dbnames.into_iter().collect(); + for dbname in dbnames { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('datatable_instance_database:' || $1))") + .bind(dbname) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + +/// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that +/// has to resolve or name a role may call it — including handlers open to a workspace member, who +/// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record +/// or an export. Nothing about who may call it: the credential is the whole risk, and `Debug` is +/// hand-written to redact it for the same reason. +pub async fn read_role_catalog(db: &DB) -> Result { + crate::datatable_roles_oss::read_role_catalog(db).await +} + +/// As [`read_role_catalog`], reading inside the caller's transaction so the value is the one +/// [`lock_role_catalog`] is protecting. Same disclosure contract. +pub async fn read_role_catalog_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, +) -> Result { + crate::datatable_roles_oss::read_role_catalog_tx(tx).await +} + +/// Record a role, in the caller's transaction so it commits with the `CREATE ROLE` it describes. +/// +/// Authorization: writes a generated Postgres credential. Callers MUST restrict this to superadmin +/// paths and MUST hold [`lock_role_catalog`] on `tx`. +pub async fn insert_role_catalog_entry( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, + role: &InstanceDatatableRole, +) -> Result<()> { + crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, role).await +} + +/// Update a role's recorded name, login flag and password. Same contract as +/// [`insert_role_catalog_entry`]. +pub async fn update_role_catalog_entry( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, + role: &InstanceDatatableRole, +) -> Result<()> { + crate::datatable_roles_oss::update_role_catalog_entry(tx, id, role).await +} + +/// Forget a role. Same contract as [`insert_role_catalog_entry`]; run it in the transaction that +/// drops the cluster login, so the two cannot disagree. +pub async fn delete_role_catalog_entry( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, +) -> Result<()> { + crate::datatable_roles_oss::delete_role_catalog_entry(tx, id).await +} + +/// Resolve the role a caller named to its catalog id. A disabled role is an error rather than a +/// silent fallback: the caller asked for something the instance deliberately turned off. +pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Result<&'a str> { + let entry = catalog + .iter() + .find(|(_, role)| role.name == name) + .ok_or_else(|| { + Error::NotFound(format!( + "'{name}' is not a data table role of this instance. Defined roles: {}.", + catalog + .values() + .map(|r| r.name.as_str()) + .collect::>() + .join(", ") + )) + })?; + if !entry.1.enabled { + return Err(Error::BadRequest(format!( + "Data table role '{name}' is disabled on this instance" + ))); + } + Ok(entry.0.as_str()) +} + +/// Every instance database the registry knows about. Role provisioning has to reach all of them: +/// a role that cannot `CONNECT` to a database is refused by Postgres before any grant matters. +/// +/// Authorization: checks nothing, and names every instance database across all workspaces. Callers +/// MUST be superadmin-gated or keep the names server-side; never return them to a workspace caller. +pub async fn registered_instance_databases(db: &DB) -> Result> { + crate::datatable_roles_oss::registered_instance_databases(db).await +} + +/// `CONNECT` on `dbname` for every enabled role, and none for `PUBLIC`. Run at role creation, at +/// database creation, and lazily whenever an instance data table is administered, so a database +/// provisioned before a role existed is repaired rather than left silently unreachable. +/// +/// Authorization: rewrites a database's ACL with the server's own credentials and checks nothing. +/// Callers MUST have authorized administration of `dbname` — superadmin, or an admin of the +/// workspace governing a data table on it. +pub async fn converge_connect_grants(db: &DB, dbname: &str) -> Result<()> { + crate::datatable_roles_oss::converge_connect_grants(db, dbname).await +} + +/// As [`converge_connect_grants`], with a catalog the caller already read. Same contract. +pub async fn converge_connect_grants_with( + db: &DB, + dbname: &str, + catalog: &DatatableRoleCatalog, +) -> Result<()> { + crate::datatable_roles_oss::converge_connect_grants_with(db, dbname, catalog).await +} + +/// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user`, and `CONNECT` on +/// every registered database. No privileges beyond that — an admin grants them through SQL or the +/// ACL editor. +/// +/// Authorization: creates a cluster-wide Postgres login. Callers MUST restrict this to superadmin +/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. +pub async fn create_instance_role( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + name: &str, + password: &str, +) -> Result<()> { + crate::datatable_roles_oss::create_instance_role(tx, name, password).await +} + +/// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin +/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. +pub async fn set_instance_role_login( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + name: &str, + enabled: bool, +) -> Result<()> { + crate::datatable_roles_oss::set_instance_role_login(tx, name, enabled).await +} + +/// A rename discards an md5-hashed password, so the caller has to hand over a fresh one. +/// +/// Authorization: renames a cluster-wide Postgres login. Callers MUST restrict this to superadmin +/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. +pub async fn rename_instance_role( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + from: &str, + to: &str, + password: &str, +) -> Result<()> { + crate::datatable_roles_oss::rename_instance_role(tx, from, to, password).await +} + +/// A role owning anything in any database blocks its own `DROP ROLE`, and both its objects and the +/// privileges granted to it are only visible from inside each database — hence the pass over the +/// registry. An unreachable database aborts the whole delete: dropping the role while one database +/// still holds objects owned by it leaves those objects owned by a numeric OID nobody can name. +/// +/// Each pass runs as the instance's own Postgres user rather than `custom_instance_user`, which +/// owns the databases and can therefore revoke a grant whoever made it. `custom_instance_user` +/// could only undo what it granted itself, so a privilege planted by an operator in psql — the +/// ordinary way privileges reach a role — would survive and block the drop. +/// +/// Authorization: drops a cluster-wide Postgres login and reassigns everything it owns. Callers +/// MUST restrict this to superadmin paths, and MUST hold [`lock_role_catalog`] on `tx`. +/// +/// The per-database passes open their own connections and cannot join `tx`; the lock is what keeps +/// a concurrent mutation out while they run. Only the final `DROP ROLE` is on `tx`, so it commits +/// or rolls back with the catalog write that forgets the role. Those passes commit as they go, so +/// callers MUST have disabled the role in an earlier committed transaction: a failure part-way +/// then leaves a disabled role to retry, not an enabled one already stripped in some databases. +pub async fn drop_instance_role( + db: &DB, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + name: &str, +) -> Result<()> { + crate::datatable_roles_oss::drop_instance_role(db, tx, name).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn role_names_are_validated() { + assert!(validate_role_name("analytics").is_ok()); + assert!(validate_role_name("read-only_2").is_ok()); + assert!(validate_role_name("").is_err()); + assert!(validate_role_name(&"a".repeat(64)).is_err()); + assert!(validate_role_name("has space").is_err()); + assert!(validate_role_name("quote\"injection").is_err()); + // Reserved, case-insensitively. + assert!(validate_role_name("admin").is_err()); + assert!(validate_role_name("Postgres").is_err()); + assert!(validate_role_name("pg_read_all_data").is_err()); + assert!(validate_role_name("windmill_user").is_err()); + assert!(validate_role_name("custom_instance_user").is_err()); + } + + #[test] + fn a_disabled_role_is_an_error_not_a_fallback() { + let mut catalog = DatatableRoleCatalog::new(); + catalog.insert( + "id1".to_string(), + InstanceDatatableRole { + name: "analytics".to_string(), + enabled: false, + pwd: Some("x".to_string()), + }, + ); + assert!(role_id_by_name(&catalog, "analytics").is_err()); + assert!(role_id_by_name(&catalog, "nope").is_err()); + catalog.get_mut("id1").unwrap().enabled = true; + assert_eq!(role_id_by_name(&catalog, "analytics").unwrap(), "id1"); + } +} diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs new file mode 100644 index 0000000000..2a3f9dda48 --- /dev/null +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -0,0 +1,209 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where data table roles come from: the enterprise implementation, or a refusal. +//! +//! Roles are an Enterprise Edition feature. An edition without them creates, grants and connects +//! as none, and a data table saved under roles — by an enterprise build, before a downgrade — is +//! refused rather than resolved as `admin`. A data table not under roles, asked for no role, +//! resolves as it always has. `private` alone is not that edition: community builds carry it. + +use crate::error::Error; + +/// What every roles path answers without the Enterprise Edition. The frontend matches this exact +/// sentence (`datatableUsableRoles.ts`) to read the refusal as "not under roles": reword both. +pub fn datatable_roles_unavailable() -> Error { + Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_roles_ee::{ + can_use_datatable_role, can_use_datatable_role_in_governing_workspace, converge_connect_grants, + converge_connect_grants_with, create_instance_role, delete_role_catalog_entry, + drop_instance_role, ensure_can_use_datatable_role, ensure_datatable_admin_access, + ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere, + insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx, + registered_instance_databases, rename_instance_role, resolve_datatable_role_connection, + set_instance_role_login, update_role_catalog_entry, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use super::datatable_roles_unavailable as unavailable; + use crate::{ + datatable_roles::{DatatableRoleCatalog, InstanceDatatableRole}, + db::AuthedRef, + error::Result, + workspaces::{ + resolve_governing_datatable, DataTableRoleTenants, DatatableAccess, GoverningDatatable, + }, + DB, + }; + + type Tx<'a> = sqlx::Transaction<'a, sqlx::Postgres>; + + pub(crate) async fn read_role_catalog(_db: &DB) -> Result { + Err(unavailable()) + } + + pub(crate) async fn read_role_catalog_tx(_tx: &mut Tx<'_>) -> Result { + Err(unavailable()) + } + + pub(crate) async fn insert_role_catalog_entry( + _tx: &mut Tx<'_>, + _id: &str, + _role: &InstanceDatatableRole, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn update_role_catalog_entry( + _tx: &mut Tx<'_>, + _id: &str, + _role: &InstanceDatatableRole, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn delete_role_catalog_entry(_tx: &mut Tx<'_>, _id: &str) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn registered_instance_databases(_db: &DB) -> Result> { + Err(unavailable()) + } + + /// Nothing to converge: with no roles to admit, an instance database keeps the `CONNECT` + /// grants it was created with, `PUBLIC`'s included, as it did before roles existed. + pub(crate) async fn converge_connect_grants(_db: &DB, _dbname: &str) -> Result<()> { + Ok(()) + } + + /// As [`converge_connect_grants`]. + pub(crate) async fn converge_connect_grants_with( + _db: &DB, + _dbname: &str, + _catalog: &DatatableRoleCatalog, + ) -> Result<()> { + Ok(()) + } + + pub(crate) async fn create_instance_role( + _tx: &mut Tx<'_>, + _name: &str, + _password: &str, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn set_instance_role_login( + _tx: &mut Tx<'_>, + _name: &str, + _enabled: bool, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn rename_instance_role( + _tx: &mut Tx<'_>, + _from: &str, + _to: &str, + _password: &str, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn drop_instance_role(_db: &DB, _tx: &mut Tx<'_>, _name: &str) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn ensure_instance_db_grant_options_unchecked( + _db: &DB, + _dbname: &str, + ) -> Result<()> { + Err(unavailable()) + } + + /// No tenant list covers anyone: there is no role to connect as. + pub(crate) fn can_use_datatable_role( + _tenants: &DataTableRoleTenants, + _authed: &AuthedRef<'_>, + ) -> bool { + false + } + + pub(crate) async fn can_use_datatable_role_in_governing_workspace( + _db: &DB, + _governing_w_id: &str, + _w_id: &str, + _tenants: &DataTableRoleTenants, + _access: &DatatableAccess<'_>, + ) -> Result { + Err(unavailable()) + } + + /// Reached only for a data table under roles or a caller naming a role: both are refused. + pub(crate) async fn resolve_datatable_role_connection( + _db: &DB, + _w_id: &str, + _name: &str, + _governing: &GoverningDatatable, + _db_resource: serde_json::Value, + _role: Option<&str>, + _access: DatatableAccess<'_>, + ) -> Result { + Err(unavailable()) + } + + /// A data table not under roles, asked for no role or for `admin`, is not a role decision and + /// passes, as it did before roles existed. Anything else is refused. + pub(crate) async fn ensure_can_use_datatable_role( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + _access: &DatatableAccess<'_>, + _context: &str, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_none() + && role.is_none_or(|r| r == crate::datatable_roles::ADMIN_DATATABLE_ROLE) + { + Ok(()) + } else { + Err(unavailable()) + } + } + + /// A data table not under roles is the `admin` connection for anyone who reaches it, as before + /// roles existed. One under roles is refused. + pub(crate) async fn ensure_datatable_admin_access( + db: &DB, + w_id: &str, + name: &str, + _access: &DatatableAccess<'_>, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + pub(crate) async fn forget_datatable_role_everywhere( + _tx: &mut Tx<'_>, + _role_id: &str, + ) -> Result<()> { + Err(unavailable()) + } +} diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 8fb124a33f..157a61f129 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the @@ -33,8 +35,8 @@ //! Every `pub` mutator in this module — the manifest ones //! (`replace_dbt_manifest`, `clear_dbt_manifest_version`, //! `clear_dbt_editor_graphs`), -//! the snapshot sweep, and the retry-state ones (`move_dbt_run_state`, -//! `clear_dbt_run_state`, `clear_dbt_run_state_if_path_retired`) — takes the +//! the snapshot sweep, and the script-state ones (`move_dbt_script_state`, +//! `clear_dbt_script_state`, `clear_dbt_script_state_if_path_retired`) — takes the //! workspace and the script to act on as plain arguments and enforces nothing: //! **the caller must already have verified write access to that script**, //! exactly like the sibling `assets::replace_static_asset_usage` each is called @@ -188,6 +190,19 @@ fn graph_digest(ingested: &IngestedManifest, relation_root: &str) -> String { .unwrap_or_default() .as_bytes(), ); + // Only when there are any, so a project that never asked for the analysis + // pass keeps the digest it already has. Hashing an empty section + // unconditionally would change every stored digest at once, and every + // dynamic run would then store a full snapshot until its script is + // redeployed — which reads exactly like the suppression above never working. + if !ingested.column_edges.is_empty() { + h.update(b"\0"); + h.update( + serde_json::to_string(&ingested.column_edges) + .unwrap_or_default() + .as_bytes(), + ); + } format!("{:x}", h.finalize()) } @@ -297,6 +312,14 @@ pub async fn prune_dbt_run_graphs( ) .execute(db) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE job_id <> '00000000-0000-0000-0000-000000000000' + AND ingested_at < now() - make_interval(days => $1)", + RUN_GRAPH_RETENTION_DAYS, + ) + .execute(db) + .await?; // In ONE transaction with the orphan sweep: a restart in the gap leaves graph // rows whose marker is gone, and since the sweep runs only when a marker went, // every later call computes `retired == 0` and skips them for good. @@ -325,7 +348,7 @@ pub async fn prune_dbt_run_graphs( // partial index here — all of them `WHERE job_id <> DEPLOYED` — and past the // keep-count is rare, so the ordinary run should pay for neither. if retired > 0 { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} t WHERE t.workspace_id = $1 AND t.script_path = $2 @@ -381,6 +404,18 @@ pub struct IngestedNode { pub severity: Option, pub attached_node: Option, pub columns: Option, + /// The node's real columns, typed and ordered — `[{"name": …, "type": …}]`, + /// from the engine's static analysis. `None` when the project did not ask + /// for it or the engine wrote none. Beside `columns` rather than merged into + /// it: that one is what the author DECLARED, and stays that. + /// + /// Skipped when absent, unlike its neighbours, because `graph_digest` + /// serializes these nodes: emitting `"column_schema":null` would change + /// every stored digest at once, and every dynamic run of a project that + /// never asked for the pass would store a full snapshot until its script is + /// redeployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub column_schema: Option, pub freshness: Option, /// The transform itself, for the graph to render. The copy taken at /// deploy: the file itself is in the script's module bundle. @@ -388,6 +423,40 @@ pub struct IngestedNode { pub original_file_path: Option, } +/// One column-to-column edge of the ingested graph. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[serde(default)] +pub struct IngestedColumnEdge { + pub parent_unique_id: String, + pub parent_column: String, + pub child_unique_id: String, + pub child_column: String, + /// dbt's own word: `copy`, `mod` or `scan`. Kept verbatim — the engine's own + /// reader maps those three and passes anything else through, so the set is + /// open. + pub lineage_kind: String, +} + +/// One column of a node, as the engine's static analysis resolved it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedColumn { + pub name: String, + /// The declared type where `schema.yml` gives one, else the inferred one. + /// Empty when neither is known; the column still belongs to the relation, so + /// only the type is left out. + pub column_type: String, + /// Position in the relation, which is the order the panel lists them in. + pub index: i64, +} + +/// What one `--write-index` pass produced: the column edges of the whole +/// project and the real column schema per node. +#[derive(Debug, Default)] +pub struct ColumnIndex { + pub edges: Vec, + pub columns: HashMap>, +} + // Serde: an agent worker cannot write these tables directly, so it posts the // whole manifest to the server, which stores it with the same function the SQL // path uses. @@ -398,6 +467,9 @@ pub struct IngestedNode { pub struct IngestedManifest { pub nodes: Vec, pub edges: Vec<(String, String)>, + /// Column-to-column lineage, when the project asked for it and the engine + /// produced it. Empty is the normal case — see `attach_column_index`. + pub column_edges: Vec, /// The `asset` rows the owning script produces (models) and consumes /// (sources) — what the lineage graph is drawn from. pub assets: Vec, @@ -405,6 +477,88 @@ pub struct IngestedManifest { pub adapter_type: String, } +/// The most column edges one graph stores. +/// +/// A `scan` edge — the column was read to produce the row, not the value — is +/// emitted from every join key and every predicate column to every output +/// column, so one wide model over a multi-column join contributes columns times +/// predicates edges on its own. The cap is what keeps a project shaped like that +/// from turning one deploy into a multi-million-row insert; past it the lineage +/// is truncated and the rest of the graph is unaffected. +pub const MAX_COLUMN_EDGES: usize = 200_000; + +/// Whether the value travelled along this edge, as opposed to the column merely +/// being read to produce the row. +/// +/// A `scan` edge reaches every output column of its model, so it is most of what +/// a wide project's index holds and the first thing `MAX_COLUMN_EDGES` gives up. +/// It is still stored, for a view that wants indirect influence. +pub fn is_direct(lineage_kind: &str) -> bool { + matches!(lineage_kind, "copy" | "mod") +} + +impl IngestedManifest { + /// Fold one `--write-index` pass into the graph. + /// + /// Both halves are scoped to the nodes this graph already kept: the index + /// describes the whole project, while the graph describes what this script's + /// selection builds plus the parents anchoring its edges, and an edge whose + /// endpoint is absent has nothing to draw. + pub fn attach_column_index(&mut self, index: ColumnIndex) { + let kept: std::collections::HashSet<&str> = + self.nodes.iter().map(|n| n.unique_id.as_str()).collect(); + let mut edges: Vec = index + .edges + .into_iter() + .filter(|e| { + kept.contains(e.parent_unique_id.as_str()) + && kept.contains(e.child_unique_id.as_str()) + }) + .collect(); + // Sorted and deduplicated for the digest, which decides whether a run + // stores a snapshot at all: parquet row order is the engine's and two + // passes over one project must not read as two different graphs. + // + // Direct kinds first, so what the truncation below gives up is `scan` — + // the bulk of a wide project's lineage, and the kind that says the column + // was read to produce the row rather than the value. The + // worker's reader already applies this order while decoding, because the + // memory bound has to; repeating it here is what makes the ordering a + // property of the manifest rather than of one caller's reader, and it is + // the only ordering an index assembled some other way would get. + edges.sort_by(|a, b| { + is_direct(&b.lineage_kind) + .cmp(&is_direct(&a.lineage_kind)) + .then_with(|| a.cmp(b)) + }); + edges.dedup(); + edges.truncate(MAX_COLUMN_EDGES); + self.column_edges = edges; + + let mut columns = index.columns; + for node in self.nodes.iter_mut() { + let Some(mut cols) = columns.remove(&node.unique_id) else { + continue; + }; + if cols.is_empty() { + continue; + } + cols.sort_by_key(|c| c.index); + node.column_schema = Some(serde_json::Value::Array( + cols.into_iter() + // A column the analysis typed as nothing still belongs in + // the list — that it exists is the half `manifest.json` + // could not answer. + .map(|c| match c.column_type.is_empty() { + true => serde_json::json!({ "name": c.name }), + false => serde_json::json!({ "name": c.name, "type": c.column_type }), + }) + .collect(), + )); + } + } +} + /// dbt's `materialized` mapped onto Windmill's write strategy. /// /// The mapping is exact for the four strategies Windmill has, and deliberately @@ -655,6 +809,9 @@ pub fn ingest_manifest( .map(|(k, v)| (k.clone(), v.description.clone().unwrap_or_default())) .collect::>()) }), + // Filled by `attach_column_index` when the project asked for it: + // the manifest carries declared columns only. + column_schema: None, freshness: node.freshness.clone(), // The transform the graph renders. Capped: a project can hold // thousands of models and this is duplicated per deploy, so a @@ -826,6 +983,16 @@ pub async fn replace_dbt_manifest( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2 + AND script_hash = $3 AND job_id = $4", + workspace_id, + script_path, + script_hash, + job_id + ) + .execute(&mut **tx) + .await?; // The marker, before the rows: a graph with no nodes at all is a legitimate // answer for a dynamic run that disabled every model, and the reader must be // able to tell it from a run that stored nothing. @@ -881,7 +1048,7 @@ async fn insert_graph_rows( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, \ resource_type, name, asset_path, materialized, materialize_strategy, unique_key, \ tags, description, test_kind, test_column, test_args, severity, attached_node, \ - columns, freshness, raw_code, original_file_path) ", + columns, column_schema, freshness, raw_code, original_file_path) ", ); q.push_values(chunk, |mut b, n| { b.push_bind(workspace_id) @@ -903,6 +1070,7 @@ async fn insert_graph_rows( .push_bind(&n.severity) .push_bind(&n.attached_node) .push_bind(&n.columns) + .push_bind(&n.column_schema) .push_bind(&n.freshness) .push_bind(&n.raw_code) .push_bind(&n.original_file_path); @@ -926,6 +1094,26 @@ async fn insert_graph_rows( q.push(" ON CONFLICT DO NOTHING"); q.build().execute(&mut **tx).await?; } + + for chunk in ingested.column_edges.chunks(COLUMN_EDGE_INSERT_CHUNK) { + let mut q = sqlx::QueryBuilder::new( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, \ + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind) ", + ); + q.push_values(chunk, |mut b, e| { + b.push_bind(workspace_id) + .push_bind(script_path) + .push_bind(script_hash) + .push_bind(job_id) + .push_bind(&e.parent_unique_id) + .push_bind(&e.parent_column) + .push_bind(&e.child_unique_id) + .push_bind(&e.child_column) + .push_bind(&e.lineage_kind); + }); + q.push(" ON CONFLICT DO NOTHING"); + q.build().execute(&mut **tx).await?; + } Ok(()) } @@ -966,7 +1154,7 @@ pub async fn replace_dbt_editor_graph( ) -> Result<()> { // By job alone, so re-executing one — a zombie recovered onto another // worker — replaces its rows rather than colliding with them. - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL" )) @@ -1016,7 +1204,7 @@ pub async fn replace_dbt_editor_graph( .fetch_all(&mut **tx) .await?; if !retired.is_empty() { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = ANY($2) \ AND script_hash IS NULL" @@ -1035,6 +1223,8 @@ pub async fn replace_dbt_editor_graph( const NODE_INSERT_CHUNK: usize = 2000; /// Six columns, so the same ceiling allows far more. const EDGE_INSERT_CHUNK: usize = 8000; +/// Nine columns, and by far the most numerous rows of the three. +const COLUMN_EDGE_INSERT_CHUNK: usize = 6000; /// Clear one VERSION's graph: the delete-by-hash route, which only soft-deletes /// its `script` row and so fires no cascade, and the ingest that finds no @@ -1070,6 +1260,15 @@ pub async fn clear_dbt_manifest_version( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + workspace_id, + script_path, + script_hash + ) + .execute(&mut **tx) + .await?; // The marker too, and every job's: a marker left standing for rows that are // gone is read as a snapshot, and its digest still answers the suppression // check — so an identical run would write nothing and then render an empty @@ -1105,7 +1304,7 @@ pub async fn clear_dbt_editor_graphs( workspace_id: &str, script_path: &str, ) -> Result<()> { - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL" @@ -1118,22 +1317,27 @@ pub async fn clear_dbt_editor_graphs( Ok(()) } -/// Move a dbt script's saved retry state to its new path. +/// Move a dbt script's saved state to its new path: the run `dbt retry` resumes, +/// and the state each environment's deferrals resolve through. /// -/// Keyed by path like the sidecar, but unlike the sidecar it is not -/// regenerated by anything: the deploy re-ingests a manifest, while these are -/// the results of a run that already happened. Clearing on rename would throw -/// away a resumable failure for a cosmetic change, so it travels instead. +/// Keyed by path like the sidecar, but unlike the sidecar neither is regenerated +/// by anything: the deploy re-ingests a manifest, while these are the results of +/// runs that already happened. Clearing on rename would throw away a resumable +/// failure, and every deferral until the next full run, for a cosmetic change — +/// so they travel instead. An artifact too large for its row is unaffected: its +/// key is that publication's own, and the moved row is what names it. /// /// See the mutator contract above: this authorizes nothing. -pub async fn move_dbt_run_state( +pub async fn move_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, old_path: &str, new_path: &str, ) -> Result<()> { // The destination may already hold state from a script that lived there - // before; the incoming row is the newer truth for this project. + // before; the incoming row is the newer truth for this project. What the + // displaced row named in object storage is left there, as a cleared one's is + // — see `clear_dbt_script_state`. sqlx::query!( "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", workspace_id, @@ -1149,22 +1353,38 @@ pub async fn move_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + new_path + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE dbt_environment_state SET script_path = $3 + WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + old_path, + new_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop the saved retry state, but only once NO live version of the path is -/// left. +/// Drop the saved state, but only once NO live version of the path is left. /// -/// `dbt_run_state`'s key is the path and the principal — one saved run per script -/// per identity it executes as, not -/// one per version — so archiving or deleting a single version must not take it -/// with them: the live version's `dbt retry` would be refused and the -/// partial-failure resume lost. It does not need to be version-scoped either, -/// because `identity` already refuses a resume whose project, warehouse or -/// engine moved. +/// Neither table is keyed by version — `dbt_run_state` by path and principal, +/// `dbt_environment_state` by path and environment — so archiving or deleting a +/// single version must not take them with it: the live version's `dbt retry` +/// would be refused, its partial-failure resume lost, and every deferral would +/// have to wait for another full run to republish. Neither needs to be +/// version-scoped either: `identity` already refuses a resume whose project, +/// warehouse or engine moved, and a deferral resolves relation names, which a +/// new version of the same project spells the same way. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state_if_path_retired( +pub async fn clear_dbt_script_state_if_path_retired( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1179,17 +1399,34 @@ pub async fn clear_dbt_run_state_if_path_retired( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2 + AND NOT EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false)", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop a dbt script's saved retry state. +/// Drop a dbt script's saved state, both halves. /// -/// Archive and delete: `run_results` is not small, the invocation arguments it -/// carries are the user's, and a script later created at the same path would -/// otherwise inherit a stranger's resumable failure. +/// Archive and delete: neither is small, the invocation arguments and manifest +/// they carry are the user's, and a script later created at the same path would +/// otherwise inherit a stranger's resumable failure and defer to a project it +/// has nothing to do with. +/// +/// An artifact too large for its row lives in the instance's object storage, and +/// this leaves it there — as a deleted script leaves its bundle. Reaching it from +/// here would mean an object-store client in this crate and a delete that has to +/// land after the caller's transaction commits, for one object per environment of +/// a script that is gone. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state( +pub async fn clear_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1201,6 +1438,13 @@ pub async fn clear_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } @@ -1765,6 +2009,69 @@ mod tests { assert_eq!(back.assets.len(), ingested.assets.len()); assert_eq!(back.assets[0].path, ingested.assets[0].path); } + + // The index describes the whole PROJECT while the graph describes what this + // script's selection builds, so an edge whose endpoint the graph does not + // hold has nothing to draw and must not be stored. + #[test] + fn column_lineage_is_scoped_to_the_nodes_the_graph_kept() { + let mut i = ingested(); + let kept = "model.jaffle_shop.customers"; + let dropped = "model.other_project.elsewhere"; + i.attach_column_index(ColumnIndex { + edges: vec![ + edge("model.jaffle_shop.orders_daily", "id", kept, "id", "copy"), + edge(dropped, "id", kept, "id", "copy"), + edge(kept, "id", dropped, "id", "copy"), + ], + columns: [ + ( + kept.to_string(), + vec![ + col("total", "Float64", 1), + col("id", "Int32", 0), + col("untyped", "", 2), + ], + ), + (dropped.to_string(), vec![col("id", "Int32", 0)]), + ] + .into(), + }); + assert_eq!( + i.column_edges + .iter() + .map(|e| (e.parent_unique_id.as_str(), e.child_unique_id.as_str())) + .collect::>(), + vec![("model.jaffle_shop.orders_daily", kept)] + ); + // In `column_index` order, and a column the analysis could not type still + // belongs to the relation. + assert_eq!( + node(&i, kept).column_schema, + Some(serde_json::json!([ + {"name": "id", "type": "Int32"}, + {"name": "total", "type": "Float64"}, + {"name": "untyped"}, + ])) + ); + assert!(node(&i, "model.jaffle_shop.orders_daily") + .column_schema + .is_none()); + } + + fn edge(from: &str, from_col: &str, to: &str, to_col: &str, kind: &str) -> IngestedColumnEdge { + IngestedColumnEdge { + parent_unique_id: from.into(), + parent_column: from_col.into(), + child_unique_id: to.into(), + child_column: to_col.into(), + lineage_kind: kind.into(), + } + } + + fn col(name: &str, column_type: &str, index: i64) -> IndexedColumn { + IndexedColumn { name: name.into(), column_type: column_type.into(), index } + } } /// Record one model's state for THIS RUN. diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 0d5ebdc2c3..da9f5f3df2 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -272,7 +272,10 @@ fn format_db_error(message: &str, detail: Option<&str>, hint: Option<&str>) -> S msg } -fn error_source_chain(e: &dyn std::error::Error) -> String { +/// `e` followed by each of its sources, `: `-separated. The result is meant for +/// users, and a `reqwest::Error` renders its request URL: never pass one built +/// from a URL carrying credentials in its userinfo. +pub fn error_source_chain(e: &dyn std::error::Error) -> String { let mut msg = e.to_string(); let mut source = e.source(); while let Some(cause) = source { diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 21b1f56389..91bfa4a39f 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -1,12 +1,39 @@ +use std::collections::HashMap; + use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; use sqlx::{self, FromRow}; use uuid::Uuid; +use windmill_types::s3::S3Object; use crate::db::DB; use crate::error::Result; use crate::utils::truncate_with_ellipsis; +/// Changing it detaches every memory stored under a string memory id. +const MEMORY_ID_NAMESPACE: Uuid = Uuid::from_u128(0x6f1c2d4e_8a3b_5c7d_9e0f_1a2b3c4d5e6f); + +/// Memory is stored and carried in `flow_status.memory_id` as a uuid, which names the same memory +/// wherever it is passed, as a chat conversation id must. Any other string names a memory through a +/// name-based (v5) uuid scoped to its workspace and flow, so the same key in two flows or two +/// workspaces names two memories, and chat conversation ids stay unique across workspaces. +pub fn memory_key(workspace_id: &str, flow_path: &str, memory_id: &str) -> Uuid { + let memory_id = memory_id.trim(); + Uuid::parse_str(memory_id).unwrap_or_else(|_| { + use sha1::{Digest, Sha1}; + let mut hasher = Sha1::new(); + hasher.update(MEMORY_ID_NAMESPACE.as_bytes()); + for part in [workspace_id, flow_path, memory_id] { + hasher.update(part.as_bytes()); + hasher.update([0u8]); + } + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&hasher.finalize()[..16]); + uuid::Builder::from_sha1_bytes(bytes).into_uuid() + }) +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, sqlx::Type)] #[sqlx(type_name = "MESSAGE_TYPE", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] @@ -26,8 +53,12 @@ pub struct FlowConversation { pub created_at: DateTime, pub updated_at: DateTime, pub created_by: String, + /// Started from the flow editor's test panel rather than a deployed run. + pub is_test: bool, } +/// `is_test` is written on insert. An existing conversation of the other kind refuses the +/// turn, so preview and deployed runs never share one. pub async fn get_or_create_conversation_with_id( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, w_id: &str, @@ -35,42 +66,140 @@ pub async fn get_or_create_conversation_with_id( username: &str, title: &str, conversation_id: Uuid, + is_test: bool, ) -> Result { - // Check if conversation already exists - let existing_conversation = sqlx::query_as!( - FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by - FROM flow_conversation - WHERE id = $1 AND workspace_id = $2", - conversation_id, - w_id - ) - .fetch_optional(&mut **tx) - .await?; - - if let Some(existing) = existing_conversation { - return Ok(existing); + if let Some(existing) = lock_conversation(tx, w_id, conversation_id).await? { + return same_kind(existing, is_test); } // Truncate title to 25 characters max let title = truncate_with_ellipsis(title, 25); - // Create new conversation with provided ID - let conversation = sqlx::query_as!( + // Every turn released by the same collector's commit finds no row: the first insert + // wins, the others wait on it, do nothing, and read the row it created. + let created = sqlx::query_as!( FlowConversation, - "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) - VALUES ($1, $2, $3, $4, $5) - RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title, is_test) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO NOTHING + RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test", conversation_id, w_id, flow_path, username, - title + title, + is_test ) - .fetch_one(&mut **tx) + .fetch_optional(&mut **tx) .await?; + if let Some(conversation) = created { + return Ok(conversation); + } - Ok(conversation) + // The concurrent first turn that won the insert may have been of the other kind. + let existing = lock_conversation(tx, w_id, conversation_id) + .await? + .ok_or_else(|| { + crate::error::Error::BadRequest(format!( + "conversation {conversation_id} belongs to another workspace" + )) + })?; + same_kind(existing, is_test) +} + +/// `memory_id` is the caller's to choose, so a preview run could name a deployed +/// conversation and the reverse. A conversation's kind is fixed at creation and nothing +/// would show the mixing afterwards, so the turn is refused before it starts. +fn same_kind(existing: FlowConversation, is_test: bool) -> Result { + if existing.is_test == is_test { + return Ok(existing); + } + Err(crate::error::Error::BadRequest(if existing.is_test { + "this conversation was started from the flow editor's test panel; start a new conversation to run the deployed flow".to_string() + } else { + "this conversation belongs to the deployed flow; start a new conversation to test from the flow editor".to_string() + })) +} + +/// Locked, so a turn orders against retention collecting the conversation +/// (windmill_common::jobs::delete_jobs): either the turn goes first and the collector then +/// sees its message, or it waits and finds the row gone and creates it again. Unlocked, the +/// message insert would wait on the parent row's lock instead and then fail its FK check. +async fn lock_conversation( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + conversation_id: Uuid, +) -> Result> { + Ok(sqlx::query_as!( + FlowConversation, + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by, is_test + FROM flow_conversation + WHERE id = $1 AND workspace_id = $2 + FOR UPDATE", + conversation_id, + w_id + ) + .fetch_optional(&mut **tx) + .await?) +} + +/// What a row carries beyond its text. A chat is rebuilt from its rows alone, without +/// reading jobs, so every tool row carries the model's call and what the model got back: a +/// Windmill tool's job holds the args its input transforms produced rather than the model's, +/// and an MCP tool's call sits among every call of the turn in the agent's job. That job's +/// `reasoning` is one string for the whole turn, where the rows keep it per iteration. +#[derive(Debug, Clone, Default)] +pub struct MessageExtras { + pub tool_arguments: Option, + pub tool_result: Option, + pub reasoning: Option, + /// The files a user message carried; see `message_attachments`. + pub attachments: Vec, +} + +/// The most files a user message keeps references to; the rest are dropped. +pub const MAX_MESSAGE_ATTACHMENTS: usize = 20; + +/// A file a user message carried, as the object-storage reference its run received. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MessageAttachment { + /// The flow input that held it. + pub input: String, + pub s3: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub storage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub filename: Option, +} + +/// The files a run's args carry for its user message: every top-level input other than +/// `user_message` whose value is an object-storage reference or a list of them, in input +/// name order, capped at `MAX_MESSAGE_ATTACHMENTS`. Only the reference is kept: `presigned` +/// grants access to the file, and any other value may be the file's bytes. +pub fn message_attachments(args: &HashMap>) -> Vec { + let mut inputs: Vec<_> = args + .iter() + .filter(|(name, _)| name.as_str() != "user_message") + .collect(); + inputs.sort_by(|a, b| a.0.cmp(b.0)); + inputs + .into_iter() + .flat_map(|(name, value)| { + serde_json::from_str::(value.get()) + .map(|object| vec![object]) + .or_else(|_| serde_json::from_str::>(value.get())) + .unwrap_or_default() + .into_iter() + .filter(|object| !object.s3.is_empty()) + .map(move |object| MessageAttachment { + input: name.clone(), + s3: object.s3, + storage: object.storage, + filename: object.filename, + }) + }) + .take(MAX_MESSAGE_ATTACHMENTS) + .collect() } /// Add a message to a conversation using an existing transaction @@ -84,6 +213,7 @@ pub async fn add_message_to_conversation_tx( message_type: MessageType, step_name: Option<&str>, success: bool, + extras: Option<&MessageExtras>, ) -> Result<()> { // Check if conversation exists first let conversation_exists = sqlx::query!( @@ -104,14 +234,21 @@ pub async fn add_message_to_conversation_tx( // Insert the message sqlx::query!( - "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success) - VALUES ($1, $2, $3, $4, $5, $6)", + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", conversation_id, message_type as MessageType, content, job_id, step_name, - success + success, + extras.and_then(|e| e.tool_arguments.as_deref()), + extras.and_then(|e| e.tool_result.as_deref()), + extras.and_then(|e| e.reasoning.as_deref()), + extras + .map(|e| &e.attachments) + .filter(|attachments| !attachments.is_empty()) + .map(sqlx::types::Json) as Option>> ) .execute(&mut **tx) .await?; @@ -143,3 +280,67 @@ pub async fn delete_conversation_memory( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, value::to_raw_value}; + + fn args(values: serde_json::Value) -> HashMap> { + values + .as_object() + .unwrap() + .iter() + .map(|(name, value)| (name.clone(), to_raw_value(value).unwrap())) + .collect() + } + + #[test] + fn keeps_only_object_storage_references() { + let attachments = message_attachments(&args(json!({ + "user_message": { "s3": "not/an/attachment.png" }, + "avatar": { "s3": "u/a.png", "storage": "secondary", "presigned": "https://signed" }, + "files": [ + { "s3": "u/b.pdf", "filename": "b.pdf" }, + { "s3": "" } + ], + "photo": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", + "count": 3 + }))); + assert_eq!( + serde_json::to_value(&attachments).unwrap(), + json!([ + { "input": "avatar", "s3": "u/a.png", "storage": "secondary" }, + { "input": "files", "s3": "u/b.pdf", "filename": "b.pdf" } + ]) + ); + } + + #[test] + fn caps_the_references_of_one_message() { + let files: Vec<_> = (0..25) + .map(|i| json!({ "s3": format!("u/{i}.png") })) + .collect(); + let attachments = message_attachments(&args(json!({ "files": files }))); + assert_eq!(attachments.len(), MAX_MESSAGE_ATTACHMENTS); + assert_eq!(attachments.last().unwrap().s3, "u/19.png"); + } + + /// A string names a memory only within its workspace and flow; a uuid is used as is. + #[test] + fn memory_key_scopes_strings_but_not_uuids() { + let key = memory_key("ws", "f/support/triage", " customer-1 "); + assert_eq!(key, memory_key("ws", "f/support/triage", "customer-1")); + assert_ne!( + key, + memory_key("other_ws", "f/support/triage", "customer-1") + ); + assert_ne!(key, memory_key("ws", "f/sales/triage", "customer-1")); + let conversation = Uuid::from_u128(7).to_string(); + assert_eq!(memory_key("ws", "f/a", &conversation), Uuid::from_u128(7)); + assert_eq!( + memory_key("other_ws", "f/b", &conversation), + Uuid::from_u128(7) + ); + } +} diff --git a/backend/windmill-common/src/folders.rs b/backend/windmill-common/src/folders.rs index 2a6ba935b6..fa4dfdf3b7 100644 --- a/backend/windmill-common/src/folders.rs +++ b/backend/windmill-common/src/folders.rs @@ -78,8 +78,6 @@ pub async fn resolve_folder_default_on_behalf_of( else { return Ok(None); }; - // Uncached: this pair is written straight onto the runnable, where a stale address would - // contradict the principal it is stored beside. let email = crate::users::get_email_from_permissioned_as_uncached(&permissioned_as, w_id, db).await?; Ok(Some((email, permissioned_as))) diff --git a/backend/windmill-common/src/git_sync_oss.rs b/backend/windmill-common/src/git_sync_oss.rs index 5f8ba771fb..4c7de4fa7b 100644 --- a/backend/windmill-common/src/git_sync_oss.rs +++ b/backend/windmill-common/src/git_sync_oss.rs @@ -1,11 +1,16 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::git_sync_ee::*; -#[cfg(not(feature = "private"))] +#[cfg(not(all(feature = "private", feature = "enterprise")))] use sqlx::{Pool, Postgres}; use url::Url; -#[cfg(not(feature = "private"))] +/// Gated on the pair to match [`with_stored_credential`] below, whose callers +/// reach it through this facade un-gated and so depend on it. Nothing routes +/// here today (the one caller imports the enterprise item directly), so this is +/// for uniformity: the next plain caller would otherwise find no definition +/// under `private` without `enterprise`. +#[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn get_github_app_token_internal( _db: &Pool, _job_token: &str, @@ -15,6 +20,21 @@ pub async fn get_github_app_token_internal( )); } +/// Server-held git credentials are an enterprise feature, so on this build a +/// repository URL authenticates with whatever it already carries. +/// +/// Gated on the pair rather than on `private` alone: `private` does not imply +/// `enterprise`, and the callers are plain (no `#[cfg]`), so a build with one +/// and not the other would find neither this nor the enterprise definition. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn with_stored_credential( + _db: &Pool, + _w_id: &str, + url: String, +) -> crate::error::Result { + Ok(url) +} + lazy_static::lazy_static! { /// Matches a `user:password@` (or `user@`) userinfo component right after the URL scheme. static ref GIT_URL_USERINFO_RE: regex::Regex = diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b539c1c6b3..6b9aae519d 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -15,6 +15,7 @@ pub const OAUTH_SETTING: &str = "oauths"; pub const AI_CONFIG_SETTING: &str = "ai_config"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides"; +pub const SERVICE_LOG_RETENTION_SECS_SETTING: &str = "service_log_retention_secs"; /// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor /// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded /// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob). @@ -64,6 +65,10 @@ pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics"; pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics"; pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; +/// Superadmin switch over guest sessions for the whole instance, above the per-workspace +/// one. Read from the table, uncached, by the same gates that read the workspace switch; +/// the superadmin Guests list writes it through `/settings/global/{key}` by this name. +pub const GUEST_ACCESS_DISABLED_SETTING: &str = "guest_access_disabled"; pub const JOB_ISOLATION_SETTING: &str = "job_isolation"; pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing"; @@ -79,6 +84,11 @@ pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; // windmill-worker/src/ssh_executor_ee.rs. pub const SSH_EXECUTION_SETTING: &str = "ssh_execution_enabled"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; +/// Whether the instance object store stands in for a workspace without storage of its own +/// as the place its members' AI sessions are backed up to. On unless the row says `false`; +/// inert without an instance object store. +pub const AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING: &str = + "ai_sessions_instance_storage_fallback"; /// Compile a newly deployed script's binary right after its dependency job and push it /// to the instance object store, so the first run does not pay the compile. Inert unless /// instance object storage is configured — without it the binary would only ever land in @@ -92,7 +102,53 @@ pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const DISABLE_WORKSPACE_INVITE_EMAILS_SETTING: &str = "disable_workspace_invite_emails"; pub const DISABLE_PASSWORD_LOGIN_SETTING: &str = "disable_password_login"; +/// Refuse `?token=` on the MCP endpoints, leaving the `Authorization` header as the only way +/// in. A URL-borne credential ends up in browser history, proxy logs and referrers, so an +/// instance that cares sends MCP clients through the OAuth flow instead. +pub const MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING: &str = "mcp_disable_token_query_param"; +/// Ceiling, in days, on how far ahead a token minted through `POST /users/tokens/create` or +/// `POST /users/tokens/impersonate` may expire; a request asking for more, or for no +/// expiration at all, is shortened to it rather than refused. On those routes only: server-side +/// mints (webhook tokens, app embed tokens, sessions) choose a lifetime the caller never picks +/// and go straight to `create_token_internal`. Read and validated by +/// [`parse_max_token_expiration_days`]. +pub const MAX_TOKEN_EXPIRATION_DAYS_SETTING: &str = "max_token_expiration_days"; +/// Largest `max_token_expiration_days` read as a ceiling, about 2,700 years. The token form +/// applies the same bound (`frontend/src/lib/tokenExpiration.ts`) so that it and the server +/// agree on whether a ceiling exists. +pub const MAX_TOKEN_EXPIRATION_DAYS_BOUND: i64 = 1_000_000; + +/// Reads a stored `max_token_expiration_days`: `Ok(None)` when unset or cleared (null or an +/// empty string), the ceiling for a whole number of days within +/// `1..=MAX_TOKEN_EXPIRATION_DAYS_BOUND` stored as an integer, an integral float or a string of +/// digits, and an error for anything else. +/// +/// The settings API and config sync both reject the error at write time: the token routes can +/// only read an unparseable value as no ceiling, so accepting a typo would silently turn the +/// policy off. `parseMaxTokenExpirationDays` in the frontend must accept exactly the same values. +pub fn parse_max_token_expiration_days( + value: Option<&serde_json::Value>, +) -> Result, String> { + let days = match value { + None | Some(serde_json::Value::Null) => return Ok(None), + Some(serde_json::Value::String(s)) if s.trim().is_empty() => return Ok(None), + Some(serde_json::Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_f64().filter(|f| f.fract() == 0.0).map(|f| f as i64)), + Some(serde_json::Value::String(s)) => s.trim().parse::().ok(), + Some(_) => None, + }; + match days { + Some(days) if (1..=MAX_TOKEN_EXPIRATION_DAYS_BOUND).contains(&days) => Ok(Some(days)), + _ => Err(format!( + "must be a whole number of days from 1 to {MAX_TOKEN_EXPIRATION_DAYS_BOUND}, or empty for no limit" + )), + } +} pub const AUTO_LOGIN_PROVIDER_SETTING: &str = "auto_login_provider"; +/// Name of the SAML attribute or OIDC userinfo claim carrying the user's IdP groups. Unset or +/// empty leaves instance-group membership entirely to SCIM. +pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; @@ -107,8 +163,10 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; +pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; +pub const HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING: &str = "http_route_default_allowed_origins"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version"; pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; @@ -116,6 +174,112 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Instance-wide announcement rendered above every page of the app (maintenance +/// windows, incidents). Readable by any authenticated user, unlike most settings: +/// the banner exists to be shown to everyone, so it must never hold anything the +/// whole instance may not see. +pub const INSTANCE_BANNER_SETTING: &str = "instance_banner"; + +/// Ceiling on the banner message. The banner is a one-or-two-line strip above every +/// page, so anything longer is a layout accident rather than an announcement. +pub const INSTANCE_BANNER_MESSAGE_MAX_LEN: usize = 500; + +/// Ceiling on the banner's link label, which renders as a button inside that same strip. +pub const INSTANCE_BANNER_LINK_LABEL_MAX_LEN: usize = 60; + +/// Validate an [`INSTANCE_BANNER_SETTING`] value. +/// +/// The banner is the one setting rendered to every user of the instance, so its +/// shape is checked at the boundary rather than trusted from the writer: a value +/// that reaches the browser malformed breaks the layout for everyone at once. +/// +/// The link is restricted to http(s) so a stored `javascript:`/`data:` URL can +/// never become the href of an anchor every user sees. +/// +/// Only shapes that would *misrender* are rejected. An enabled banner with no message +/// is left alone deliberately: it renders as nothing, and every write path here runs +/// under the bulk settings save, so rejecting it would fail an admin's whole settings +/// edit — retention, SMTP and all — over a half-typed announcement. +pub fn validate_instance_banner(value: &serde_json::Value) -> Result<(), String> { + let obj = value + .as_object() + .ok_or_else(|| "must be a JSON object".to_string())?; + + // Field types are checked before their contents. Every read below is an `as_str`/ + // `as_bool`, which reports a wrong-typed field as absent — so without this a + // `"link": 123` would skip the URL checks entirely and be stored, and the settings + // form would then throw on it (`link.trim()` on a number) instead of rendering. + for (field, expected, ok) in [ + ( + "enabled", + "a boolean", + obj.get("enabled").is_none_or(|v| v.is_boolean()), + ), + ( + "dismissible", + "a boolean", + obj.get("dismissible").is_none_or(|v| v.is_boolean()), + ), + ( + "message", + "a string", + obj.get("message").is_none_or(|v| v.is_string()), + ), + ( + "severity", + "a string", + obj.get("severity").is_none_or(|v| v.is_string()), + ), + ( + "link", + "a string", + obj.get("link").is_none_or(|v| v.is_string()), + ), + ( + "link_label", + "a string", + obj.get("link_label").is_none_or(|v| v.is_string()), + ), + ] { + if !ok { + return Err(format!("{field} must be {expected}")); + } + } + + for (field, max) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN), + ] { + let len = obj + .get(field) + .and_then(|v| v.as_str()) + .map_or(0, |s| s.chars().count()); + if len > max { + return Err(format!("{field} must be at most {max} characters")); + } + } + + if let Some(severity) = obj.get("severity").and_then(|v| v.as_str()) { + if !matches!(severity, "info" | "warning" | "error") { + return Err("severity must be one of info, warning, error".to_string()); + } + } + + if let Some(link) = obj.get("link").and_then(|v| v.as_str()) { + if !link.trim().is_empty() { + let url = url::Url::parse(link.trim()) + .map_err(|e| format!("link must be an absolute http(s) URL: {e}"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err("link must use the http or https scheme".to_string()); + } + if !url.has_host() { + return Err("link must include a host".to_string()); + } + } + } + + Ok(()) +} /// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. /// @@ -247,6 +411,126 @@ use std::sync::atomic::AtomicBool; lazy_static::lazy_static! { pub static ref HTTP_ROUTE_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false); pub static ref DISABLE_PASSWORD_LOGIN: AtomicBool = AtomicBool::new(false); + pub static ref MCP_DISABLE_TOKEN_QUERY_PARAM: AtomicBool = AtomicBool::new(false); + /// Origins HTTP routes allow cross-origin when they configure none of their + /// own. Empty means unset, which keeps the historical `*`. + pub static ref HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS: arc_swap::ArcSwap> = + arc_swap::ArcSwap::from_pointee(vec![]); +} + +/// Whether an allowlist places no restriction at all. +/// +/// `*` is the explicit "open on purpose" entry, and a route carrying it behaves +/// exactly as an unconfigured one: it is how a route opts out of a stricter +/// instance default, including back into the `wm_headers` escape hatch. +pub fn allows_any_origin(allowed_origins: &[String]) -> bool { + allowed_origins.iter().any(|allowed| allowed == "*") +} + +/// An allowlist is scanned on every request to a restricted route, including +/// the unauthenticated preflight, so its size is a request cost anyone can +/// trigger. +pub const MAX_ALLOWED_ORIGINS: usize = 100; +pub const MAX_ALLOWED_ORIGIN_LEN: usize = 256; + +/// Reject allowlist entries that cannot be compared, stored, or safely allowed. +/// +/// The stored string is only ever an operand: `match_origin` echoes the +/// request's own `Origin` back, never this value, so a malformed entry matches +/// nothing and fails closed. Shapes that merely cannot match are the editor's +/// business to warn about, not this function's to refuse. What is left are the +/// three cases where permissiveness costs something: `null` is what every +/// sandboxed iframe sends, so allowing it would admit any page that can open +/// one; a comma cannot survive the editor's comma-separated field, which would +/// silently split one entry into two and widen the list; and an unbounded list +/// makes every preflight pay for it. +pub fn validate_allowed_origins(allowed_origins: &[String]) -> crate::error::Result<()> { + if allowed_origins.len() > MAX_ALLOWED_ORIGINS { + return Err(crate::error::Error::BadRequest(format!( + "At most {} allowed origins, got {}.", + MAX_ALLOWED_ORIGINS, + allowed_origins.len() + ))); + } + + for origin in allowed_origins { + if origin == "*" { + continue; + } + + let invalid = |reason: &str| { + crate::error::Error::BadRequest(format!( + "Invalid allowed origin '{}': {}.", + origin, reason + )) + }; + + if origin.is_empty() { + return Err(invalid("must not be empty")); + } + if origin.len() > MAX_ALLOWED_ORIGIN_LEN { + return Err(invalid("is longer than any origin a browser sends")); + } + // The editor edits the whole list as one comma-separated field, so an + // entry carrying a comma comes back as two and widens the list. + if origin.contains(',') { + return Err(invalid("must not contain a comma, which separates entries")); + } + if origin.eq_ignore_ascii_case("null") { + return Err(invalid( + "'null' is what a sandboxed iframe sends, so allowing it would allow any page that can open one", + )); + } + // An Origin header is always visible ASCII, so a value outside it can + // never be the string this is compared against. + if !origin.chars().all(|c| c.is_ascii_graphic()) { + return Err(invalid( + "must contain only visible ASCII, with no whitespace", + )); + } + } + + Ok(()) +} + +/// Read [`HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING`] from its stored value. +/// +/// Accepts the comma-separated string the settings UI writes, or a JSON array +/// for anything setting it through the API directly. +pub fn parse_allowed_origins_setting( + value: Option<&serde_json::Value>, +) -> crate::error::Result> { + let origins = match value { + None | Some(serde_json::Value::Null) => vec![], + Some(serde_json::Value::String(raw)) => raw + .split(',') + .map(|origin| origin.trim().to_string()) + .filter(|origin| !origin.is_empty()) + .collect(), + Some(serde_json::Value::Array(entries)) => entries + .iter() + .map(|entry| match entry { + serde_json::Value::String(origin) => Ok(origin.trim().to_string()), + _ => Err(crate::error::Error::BadRequest(format!( + "{} entries must be strings", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ))), + }) + // Not filtered for empties, unlike the string form: there a + // trailing separator naturally yields an empty token, whereas an + // empty array entry is something the caller wrote and validation + // should reject rather than silently drop. + .collect::>>()?, + Some(_) => { + return Err(crate::error::Error::BadRequest(format!( + "{} expected to be a comma-separated string or an array of strings", + HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING + ))) + } + }; + + validate_allowed_origins(&origins)?; + Ok(origins) } pub const ENV_SETTINGS: &[&str] = &[ @@ -318,6 +602,31 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_METRICS", "OTEL_TRACING", "OTEL_LOGS", + // The OTEL_EXPORTER_OTLP_*HEADERS variables are left out: they carry exporter API keys, and + // this list is logged at startup and returned to superadmins by `get_local_settings`. + "OTEL_METRICS_ENABLED", + "OTEL_TRACING_ENABLED", + "OTEL_LOGS_ENABLED", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_COMPRESSION", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_ENVIRONMENT", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_JOB_LOGS", + "OTEL_TRACES_RETENTION_SECS", + "AI_SHARED_ARTIFACT_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", @@ -546,6 +855,55 @@ pub fn workspace_integration_auth_endpoint(client_name: &str, base_url: &str) -> mod tests { use super::*; + // `frontend/src/lib/tokenExpiration.test.ts` holds the same table for the token form's + // parser; the two must stay in step. + #[test] + fn max_token_expiration_days_accepts_only_whole_days_within_the_bound() { + use serde_json::json; + for (stored, days) in [ + (json!(7), 7), + (json!(7.0), 7), + (json!("7"), 7), + (json!(" 30 "), 30), + (json!("+7"), 7), + ( + json!(MAX_TOKEN_EXPIRATION_DAYS_BOUND), + MAX_TOKEN_EXPIRATION_DAYS_BOUND, + ), + ] { + assert_eq!( + parse_max_token_expiration_days(Some(&stored)), + Ok(Some(days)), + "{stored}" + ); + } + for cleared in [json!(null), json!(""), json!(" ")] { + assert_eq!( + parse_max_token_expiration_days(Some(&cleared)), + Ok(None), + "{cleared}" + ); + } + assert_eq!(parse_max_token_expiration_days(None), Ok(None)); + for bad in [ + json!(7.5), + json!(0), + json!(-3), + json!("7.0"), + json!("1e1"), + json!("0x7"), + json!(MAX_TOKEN_EXPIRATION_DAYS_BOUND + 1), + json!("99999999999999999999"), + json!(true), + json!([7]), + ] { + assert!( + parse_max_token_expiration_days(Some(&bad)).is_err(), + "{bad} must be rejected" + ); + } + } + #[test] fn webhook_base_url_errors_never_echo_credentials() { // These strings reach sync-config output and operator logs, so no branch may @@ -581,6 +939,61 @@ mod tests { } } + #[test] + fn instance_banner_rejects_unsafe_and_malformed_values() { + // The link becomes the href of an anchor shown to every user of the instance, + // so a non-http(s) scheme must not survive a write. + for link in [ + "javascript:alert(1)", + "data:text/html,", + "vbscript:msgbox(1)", + "not-a-url", + "https://", + ] { + let banner = serde_json::json!({ "enabled": true, "message": "down", "link": link }); + assert!( + validate_instance_banner(&banner).is_err(), + "link '{link}' should be rejected" + ); + } + // A wrong-typed field reads as absent to every accessor here, so without an + // explicit type check it would skip validation and be stored. + for bad in [ + serde_json::json!({ "enabled": true, "message": "down", "link": 123 }), + serde_json::json!({ "enabled": true, "message": "down", "link_label": ["a"] }), + serde_json::json!({ "enabled": true, "message": { "text": "down" } }), + serde_json::json!({ "enabled": true, "message": "down", "severity": 2 }), + serde_json::json!({ "enabled": "yes", "message": "down" }), + serde_json::json!({ "enabled": true, "message": "down", "dismissible": "no" }), + ] { + assert!( + validate_instance_banner(&bad).is_err(), + "{bad} should be rejected" + ); + } + // The strip is one or two lines tall; both of its texts are bounded. + for (field, over) in [ + ("message", INSTANCE_BANNER_MESSAGE_MAX_LEN + 1), + ("link_label", INSTANCE_BANNER_LINK_LABEL_MAX_LEN + 1), + ] { + let mut banner = serde_json::json!({ "enabled": true, "message": "down" }); + banner[field] = serde_json::Value::String("x".repeat(over)); + assert!( + validate_instance_banner(&banner).is_err(), + "an over-long {field} should be rejected" + ); + } + // Enabled with no message renders as nothing and must stay writable: every path + // into this validator is a bulk settings save, so rejecting it would fail an + // admin's unrelated edits over a half-typed announcement. + assert!(validate_instance_banner(&serde_json::json!({ "enabled": true })).is_ok()); + let ok = serde_json::json!({ + "enabled": true, "message": "down", "severity": "warning", + "link": "https://status.example.com", "dismissible": false + }); + assert!(validate_instance_banner(&ok).is_ok()); + } + #[test] fn webhook_base_url_matches_the_ui_validator() { // Kept in lockstep with `isValidWebhookBaseUrl` in diff --git a/backend/windmill-common/src/guest_jwt.rs b/backend/windmill-common/src/guest_jwt.rs new file mode 100644 index 0000000000..dc0407163d --- /dev/null +++ b/backend/windmill-common/src/guest_jwt.rs @@ -0,0 +1,1076 @@ +//! The guest JWT contract: what a token minted by an embedding customer's own backend +//! must carry to open one guest-mode app, and how it is verified against the workspace's +//! configured key (or, off cloud, the instance issuer). Deliberately narrower than the external JWT scheme +//! (`jwt_ext_`), whose claims can assert admin, groups and folders: a guest key can +//! only ever mint guests, whatever the token says. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use jsonwebtoken::{ + jwk::{AlgorithmParameters, Jwk, PublicKeyUse}, + Algorithm, DecodingKey, Validation, +}; +use quick_cache::sync::Cache; +use serde::Deserialize; + +use crate::error::{Error, Result}; +use crate::DB; + +/// A token is honoured at most this long past its issue, however far its `exp` lies: +/// a guest's expiry is its only revocation, and a long-lived token minted by mistake +/// would otherwise stay valid until it leaked. +pub const MAX_LIFETIME_SECS: u64 = 24 * 60 * 60; + +/// Bearer prefix. Stateless: no `token` row. Verified against the workspace's key (or, off +/// cloud, the instance issuer when the workspace set none) and resolved in the auth cache, +/// whose entry is short-lived (not the token's full `exp`) so a rotated key revokes within +/// minutes. See the arm in `windmill-api-auth`. +pub const BEARER_PREFIX: &str = "jwt_guest_"; + +const RSA_ALGORITHMS: [Algorithm; 6] = [ + Algorithm::RS256, + Algorithm::RS384, + Algorithm::RS512, + Algorithm::PS256, + Algorithm::PS384, + Algorithm::PS512, +]; +const EC_ALGORITHMS: [Algorithm; 2] = [Algorithm::ES256, Algorithm::ES384]; + +/// Every claim honoured. Extra claims are ignored; a missing one refuses the token. +/// `app_path` is mandatory: a token opens that one app, exactly as a signed-in guest +/// session does. +#[derive(Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct GuestJwtClaims { + pub email: String, + pub workspace_id: String, + pub app_path: String, + pub exp: u64, + pub nbf: Option, + pub iat: Option, +} + +/// How a guest JWT is verified: the workspace's configured key (a PEM public key or a JWKS +/// URL), or, off cloud, the instance issuer (`JWT_EXT_JWKS_URL`) when the workspace set none — +/// see `key_source`. When neither is set the JWT is refused whatever it carries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GuestJwtKeySource { + Pem(String), + JwksUrl(String), +} + +/// The instance-wide external JWT issuer (`JWT_EXT_JWKS_URL`, also used by `jwt_ext_`). Read +/// fresh rather than cached so it is testable and picks up config regardless of init order. +fn instance_ext_jwks_url() -> Option { + std::env::var("JWT_EXT_JWKS_URL") + .ok() + .filter(|s| !s.trim().is_empty()) +} + +pub async fn key_source(db: &DB, w_id: &str) -> Result> { + let row = sqlx::query!( + "SELECT guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("reading guest JWT key of {w_id}: {e:#}")))?; + let per_workspace = row.and_then(|r| match (r.guest_jwt_public_key, r.guest_jwt_jwks_url) { + (Some(pem), _) => Some(GuestJwtKeySource::Pem(pem)), + (None, Some(url)) => Some(GuestJwtKeySource::JwksUrl(url)), + (None, None) => None, + }); + if per_workspace.is_some() { + return Ok(per_workspace); + } + // No workspace key: fall back to the instance issuer, so an operator running one issuer for + // both `jwt_ext_` and guests configures it once. Verifying it (and granting a *guest*) is + // done here in CE; granting a full login from it stays EE (`jwt_ext_`). Not on the shared + // cloud, where one instance issuer must not be trusted to mint guests in every tenant's + // workspace — there the per-workspace key is the only source. + if !*crate::worker::CLOUD_HOSTED { + if let Some(url) = instance_ext_jwks_url() { + return Ok(Some(GuestJwtKeySource::JwksUrl(url))); + } + } + Ok(None) +} + +/// Parse a PEM public key and the algorithms it may verify: RSA keys the RS/PS family, +/// EC keys the ES family. Anything symmetric has no PEM form, so HS* is unreachable +/// from here by construction; the JWKS path refuses it explicitly. +pub fn decoding_key_from_pem(pem: &str) -> Result<(DecodingKey, &'static [Algorithm])> { + use base64::{engine::general_purpose::STANDARD, Engine}; + use spki::der::Decode; + // The key is admin-set into an unbounded `TEXT` column and reparsed on every guest-JWT + // request; a well-formed key with an oversized modulus would pass the checks below. Refuse + // one larger than any real public key before decoding or storing it. Measure the untrimmed + // input: the endpoint stores what the admin sent, so whitespace padding counts too. + if pem.len() > MAX_GUEST_PEM_LEN { + return Err(Error::BadRequest(format!( + "guest key is longer than {MAX_GUEST_PEM_LEN} bytes" + ))); + } + let pem = pem.trim(); + // A verification key must be public. jsonwebtoken 8.3 keys the public/private distinction + // off the PEM label alone and never inspects the DER, so private material relabelled + // `PUBLIC KEY` would be stored and then served back through the settings response. Decode + // the body leniently, as jsonwebtoken does (tolerating any wrapping the strict RFC 7468 + // decoder would refuse), then require it to be a public-key structure: an SPKI (RSA or EC) + // or a PKCS#1 RSA public key. Private-key DER satisfies neither. + let der = STANDARD + .decode( + pem.lines() + .filter(|l| !l.trim_start().starts_with("-----")) + .flat_map(|l| l.split_whitespace()) + .collect::(), + ) + .map_err(|e| Error::BadRequest(format!("guest key is not valid PEM: {e}")))?; + let is_public = spki::SubjectPublicKeyInfoRef::from_der(&der).is_ok() + || pkcs1::RsaPublicKey::from_der(&der).is_ok(); + if !is_public { + return Err(Error::BadRequest( + "expected an RSA or EC public key in PEM form (-----BEGIN PUBLIC KEY-----)".to_string(), + )); + } + if let Ok(key) = DecodingKey::from_rsa_pem(pem.as_bytes()) { + return Ok((key, &RSA_ALGORITHMS)); + } + if let Ok(key) = DecodingKey::from_ec_pem(pem.as_bytes()) { + return Ok((key, &EC_ALGORITHMS)); + } + Err(Error::BadRequest( + "not an RSA or EC public key in PEM form (expected -----BEGIN PUBLIC KEY-----)".to_string(), + )) +} + +/// The algorithms a JWKS key may verify, or `None` if the key is unusable here: a +/// symmetric key (HS*, a shared secret the embedder would then have to hold), an +/// unsupported family, or a key not marked for signatures. A key that names its `alg` +/// pins that one; an RSA key that omits it accepts the whole RSA family, and an EC key +/// the algorithm its curve implies, mirroring how a PEM key is accepted. +pub fn jwk_algorithms(jwk: &Jwk) -> Option> { + if jwk.common.public_key_use.is_some() + && jwk.common.public_key_use != Some(PublicKeyUse::Signature) + { + return None; + } + // A key that lists its operations must allow verifying signatures; otherwise it is + // published for something else (encryption, key wrapping) and is not ours to use. + if jwk + .common + .key_operations + .as_ref() + .is_some_and(|ops| !ops.contains(&jsonwebtoken::jwk::KeyOperations::Verify)) + { + return None; + } + let alg = crate::jwt::jwk_algorithm(jwk); + // A key that names an `alg` we cannot verify with (RSA-OAEP, ...) is not an alg-less + // key to be tried against the whole family: it is published for something else. + if alg.is_none() && jwk.common.key_algorithm.is_some() { + return None; + } + match (&jwk.algorithm, alg) { + (AlgorithmParameters::RSA(_), Some(alg)) if RSA_ALGORITHMS.contains(&alg) => { + Some(vec![alg]) + } + (AlgorithmParameters::RSA(_), None) => Some(RSA_ALGORITHMS.to_vec()), + (AlgorithmParameters::EllipticCurve(_), Some(alg)) if EC_ALGORITHMS.contains(&alg) => { + Some(vec![alg]) + } + (AlgorithmParameters::EllipticCurve(p), None) => match p.curve { + jsonwebtoken::jwk::EllipticCurve::P256 => Some(vec![Algorithm::ES256]), + jsonwebtoken::jwk::EllipticCurve::P384 => Some(vec![Algorithm::ES384]), + _ => None, + }, + _ => None, + } +} + +/// Verify `token` against `key`, honouring only the accepted `algorithms`, and check +/// every claim rule that needs no database: signature, `exp` (mandatory), `nbf` and +/// `iat` when present, the lifetime cap, that the token names `w_id`, that `email` is a +/// valid address bounded to 254 bytes, and that `app_path` carries no scope metacharacter. +pub fn verify( + token: &str, + key: &DecodingKey, + algorithms: &[Algorithm], + w_id: &str, +) -> Result { + let mut validation = Validation::new(algorithms[0]); + validation.algorithms = algorithms.to_vec(); + validation.validate_nbf = true; + // Audience is not part of the guest contract: issuers commonly stamp their own `aud`, + // and no accepted audience is configured, so an `aud` claim must not fail the token. + validation.validate_aud = false; + let claims = jsonwebtoken::decode::(token, key, &validation) + .map_err(|e| Error::NotAuthorized(format!("guest JWT refused: {e}")))? + .claims; + let now = jsonwebtoken::get_current_timestamp(); + if claims.exp > now + MAX_LIFETIME_SECS { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: exp is more than {MAX_LIFETIME_SECS} seconds ahead" + ))); + } + if let Some(iat) = claims.iat { + if iat > now + validation.leeway { + return Err(Error::NotAuthorized( + "guest JWT refused: iat is in the future".to_string(), + )); + } + if claims.exp.saturating_sub(iat) > MAX_LIFETIME_SECS { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: lifetime exceeds {MAX_LIFETIME_SECS} seconds" + ))); + } + } + if claims.workspace_id != w_id { + return Err(Error::NotAuthorized( + "guest JWT refused: workspace_id does not match the workspace".to_string(), + )); + } + // The email becomes the guest's username; require the address shape the `usr` table + // accepts (`VALID_EMAIL`), so it always carries an `@` and `username_to_permissioned_as` + // can only ever read it as its own principal, never a `u/` or `g/`. + // Bound it to fit the `guest_activity.email` column: a longer one fails that insert + // while the guest is admitted uncounted. + if !crate::users::VALID_EMAIL.is_match(&claims.email) || claims.email.len() > 254 { + return Err(Error::NotAuthorized( + "guest JWT refused: email is not a valid, bounded email address".to_string(), + )); + } + // The app path is spliced into `apps:read:` and `apps:run:` scopes, whose + // grammar reserves `:`, `,`, `*` and a leading `/`; refuse those, the same guard + // `guest_session_scopes` applies at the mint. App paths may carry spaces and `@`. + if !crate::auth::is_scope_literal_path(&claims.app_path) { + return Err(Error::NotAuthorized( + "guest JWT refused: app_path is empty or cannot be scoped (`:`, `,`, `*` are \ + reserved and a leading `/` never matches a route)" + .to_string(), + )); + } + Ok(claims) +} + +struct JwksEntry { + keys: Arc>, + /// When this entry stops being served and the next request refetches. A good fetch + /// is served for `JWKS_TTL`, a failed one for `JWKS_NEGATIVE_TTL` (serving the last + /// good keys if there are any), so an unreachable issuer cannot be turned into one + /// outbound fetch per request by unauthenticated traffic. + expires_at: Instant, + /// When these keys were last fetched successfully. Stale keys are served only within + /// `JWKS_MAX_STALE` of this, and a stale re-serve preserves it, so a revoked `kid` or an + /// unreachable issuer stops minting new guest JWTs after a bounded window, not forever. + fetched_at: Instant, +} + +lazy_static::lazy_static! { + static ref JWKS_CACHE: Cache> = Cache::new(200); + /// Per-URL fetch lock: only one refresh per URL is in flight at a time, so a cold + /// or stale entry under a burst triggers one fetch, not one per request. This is a + /// plain map, not a capacity-bounded `Cache`: a `Cache` could evict a lock whose fetch + /// is still running, and the next request for that URL would then mint a fresh lock and + /// start a duplicate fetch, so cycling past 200 cold URLs could defeat single-flight and + /// storm the issuers. `JwksFetchLock` drops each entry once its last holder is gone, so + /// the map only ever holds the fetches in flight (bounded by concurrent distinct URLs). + static ref JWKS_FETCH_LOCKS: std::sync::Mutex>>> = + std::sync::Mutex::new(HashMap::new()); +} + +/// How long a good key set is served before a refresh; also the lag before a +/// rotated-in `kid` is picked up. The cadence of the instance-level external JWKS. +const JWKS_TTL: Duration = Duration::from_secs(15 * 60); +/// How long a failed fetch is remembered before retrying, so an unreachable issuer is +/// hit at most once per this interval however much guest-JWT traffic arrives. +const JWKS_NEGATIVE_TTL: Duration = Duration::from_secs(30); +/// The absolute age past which cached keys are no longer served, even while revalidating: +/// once an issuer has been unreachable (or has revoked a `kid`) for this long, its old keys +/// stop authenticating and the request fails closed rather than trusting them indefinitely. +const JWKS_MAX_STALE: Duration = Duration::from_secs(60 * 60); +/// A JWKS body larger than this is refused rather than buffered: the URL is admin-set +/// but the server it names may be attacker-controlled, and a real key set is a few KB. +const JWKS_MAX_BYTES: usize = 1 << 20; + +/// A cache entry retains only the usable signing keys, but nothing else bounds their combined +/// size: the response cap is 1 MiB and `from_jwk` decodes `n`/`e`/`x`/`y` without a length +/// limit, so one entry could retain ~1 MiB (a few hundred MB across the 200-entry LRU). Cap the +/// retained material instead; a real set is a few KB, so this is invisible to a legitimate one. +const JWKS_MAX_RETAINED_BYTES: usize = 64 * 1024; + +/// The retained-bytes cap counts key material; this caps the number of keys so the per-key +/// fixed cost (each `Jwk` and its map slot) is bounded too, not just their string content. +const JWKS_MAX_KEYS: usize = 50; + +/// Both JWKS caches key on the admin-supplied URL string. The column is unbounded `TEXT`, so +/// without this a workspace admin could grow the caches by the URL bytes alone. Enforced in +/// `fetch_jwks`, which `edit_guest_jwt_key` validates through, so an overlong URL is never +/// stored; the cache only ever sees a URL that was stored, hence a bounded one. +const MAX_JWKS_URL_LEN: usize = 2048; + +/// A guest verification key is admin-set into an unbounded `TEXT` column and reparsed on every +/// guest-JWT request. A real public key PEM is under a few KB (RSA-16384 SPKI is ~2.8 KB), so +/// this bounds the stored and reparsed bytes without refusing any real key. +const MAX_GUEST_PEM_LEN: usize = 8 * 1024; + +/// A guest JWT is refused past this before any signature work or caching: the auth cache keys +/// on the bearer, so an oversized token (unauthenticated at this point) would otherwise be +/// decoded and, if it verified, cached at its full size. A real JWT is well under this. +pub const MAX_GUEST_JWT_LEN: usize = 8 * 1024; + +/// Fetch a JWKS, keeping only the keys usable here. A workspace-admin URL is validated +/// against private ranges and the connect pinned to the validated addresses; redirects are +/// not followed for the same reason. The instance issuer (`JWT_EXT_JWKS_URL`) is exempt from +/// those restrictions — it is operator-configured and trusted (it also backs `jwt_ext_`), so a +/// self-hosted internal (http/private) issuer that works for `jwt_ext_` works for guests too. +/// The body is read with a cap so a hostile endpoint cannot exhaust memory. +pub async fn fetch_jwks(url: &str) -> Result> { + use futures::StreamExt; + if url.len() > MAX_JWKS_URL_LEN { + return Err(Error::BadRequest(format!( + "JWKS URL is longer than {MAX_JWKS_URL_LEN} bytes" + ))); + } + let resp = if instance_ext_jwks_url().as_deref() == Some(url) { + // Operator-trusted issuer: fetch it with the same permissive client `jwt_ext_` uses + // (follows redirects, honors ACCEPT_INVALID_CERTS), so an issuer that works for + // `jwt_ext_` through a redirect or an approved self-signed cert works for guests too. + crate::utils::HTTP_CLIENT_PERMISSIVE.get(url).send().await + } else { + // Workspace-admin URL: validate (https + private ranges), pin the connect to the + // validated addresses, and do not follow redirects — all against SSRF. + let client = crate::ssrf::validate_guest_jwks_url(url) + .await + .map_err(|e| Error::BadRequest(format!("JWKS URL is not allowed: {e}")))? + .apply_dns_pinning(crate::utils::configure_client(reqwest::ClientBuilder::new())) + .user_agent("windmill/beta") + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(10)) + .build() + .map_err(|e| Error::internal_err(format!("building JWKS client: {e}")))?; + client.get(url).send().await + } + .and_then(|r| r.error_for_status()) + .map_err(|e| Error::BadRequest(format!("could not fetch JWKS: {e}")))?; + let mut stream = resp.bytes_stream(); + let mut body: Vec = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| Error::BadRequest(format!("reading JWKS: {e}")))?; + if body.len() + chunk.len() > JWKS_MAX_BYTES { + return Err(Error::BadRequest(format!( + "JWKS is larger than {JWKS_MAX_BYTES} bytes" + ))); + } + body.extend_from_slice(&chunk); + } + parse_jwks_keys(&body) +} + +/// The usable signing keys in a JWKS body, by `kid`. Each key is parsed on its own and +/// one that does not model as a JWT key is skipped, not fatal: a set may legitimately +/// carry an encryption key (say `alg: "RSA-OAEP"`, which is not in jsonwebtoken's signing +/// `Algorithm` enum and would fail whole-set deserialization) beside its signing keys. +/// +/// A key is kept only if its material actually decodes (`DecodingKey::from_jwk`): jsonwebtoken +/// carries `n`/`e`/`x`/`y` as strings and defers decoding to auth time, so without this a JWKS +/// whose only key is malformed would be accepted at save time and fail every token later. +fn parse_jwks_keys(body: &[u8]) -> Result> { + let set: serde_json::Value = serde_json::from_slice(body) + .map_err(|e| Error::BadRequest(format!("JWKS is not JSON: {e}")))?; + let entries = set + .get("keys") + .and_then(|k| k.as_array()) + .ok_or_else(|| Error::BadRequest("JWKS has no `keys` array".to_string()))?; + let keys: HashMap = entries + .iter() + .filter_map(|entry| serde_json::from_value::(entry.clone()).ok()) + .filter(|jwk| jwk_algorithms(jwk).is_some()) + .filter(|jwk| DecodingKey::from_jwk(jwk).is_ok()) + .filter_map(|jwk| jwk.common.key_id.clone().map(|kid| (kid, jwk))) + .collect(); + if keys.is_empty() { + return Err(Error::BadRequest( + "JWKS holds no RSA or EC signing key with a kid".to_string(), + )); + } + // Bound the usable keys two ways, both measured after filtering so a large mixed-use set + // (many encryption keys, few signing) is not refused for its size: their count (the per-key + // fixed cost) and their combined material bytes. + if keys.len() > JWKS_MAX_KEYS { + return Err(Error::BadRequest(format!( + "JWKS holds more than {JWKS_MAX_KEYS} usable signing keys" + ))); + } + let retained: usize = keys + .values() + .filter_map(|jwk| serde_json::to_vec(jwk).ok().map(|v| v.len())) + .sum(); + if retained > JWKS_MAX_RETAINED_BYTES { + return Err(Error::BadRequest(format!( + "JWKS signing keys retain more than {JWKS_MAX_RETAINED_BYTES} bytes" + ))); + } + Ok(keys) +} + +/// A cached entry that holds no keys is a remembered failure; serving it would report +/// an unreachable issuer as an unknown `kid`. Map it to an issuer-unreachable error. +fn servable(entry: Arc) -> Result> { + if entry.keys.is_empty() { + Err(Error::NotAuthorized( + "guest JWT refused: the JWKS issuer is unreachable".to_string(), + )) + } else { + Ok(entry) + } +} + +/// A held single-flight lock for one JWKS URL. Dropping it removes the URL from +/// `JWKS_FETCH_LOCKS` once no other holder remains, so the registry never keeps a lock past +/// its fetch and stays bounded by the number of fetches in flight, not by URLs ever seen. +struct JwksFetchLock { + url: String, + lock: Arc>, +} + +impl JwksFetchLock { + /// The lock for `url`, created on first use. Callers sharing a URL get the same `Arc`, + /// so one holds the inner mutex and fetches while the rest wait on it. + fn acquire(url: &str) -> Self { + let mut map = JWKS_FETCH_LOCKS.lock().unwrap(); + let lock = map + .entry(url.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone(); + JwksFetchLock { url: url.to_string(), lock } + } +} + +impl Drop for JwksFetchLock { + fn drop(&mut self) { + let mut map = JWKS_FETCH_LOCKS.lock().unwrap(); + // Clones are only taken while holding this same map lock, so the count is stable + // here: two Arcs (the map's and this one's) means we are the last holder and the + // entry can go; more means another request still needs it and will remove it in turn. + if map + .get(&self.url) + .is_some_and(|lock| Arc::strong_count(lock) <= 2) + { + map.remove(&self.url); + } + } +} + +/// Refresh a URL's JWKS off the request path, under the single-flight lock. A held +/// lock means a refresh is already running, so this is a no-op. A failed refresh +/// leaves the served stale keys in place rather than dropping them. +fn spawn_jwks_refresh(url: String) { + tokio::spawn(async move { + let fetch_lock = JwksFetchLock::acquire(&url); + let Ok(_guard) = fetch_lock.lock.try_lock() else { + return; + }; + match fetch_jwks(&url).await { + Ok(keys) => { + let now = Instant::now(); + JWKS_CACHE.insert( + url, + Arc::new(JwksEntry { + keys: Arc::new(keys), + expires_at: now + JWKS_TTL, + fetched_at: now, + }), + ); + } + Err(e) => tracing::warn!("guest JWKS background refresh failed for {url}: {e:#}"), + } + }); +} + +/// The workspace's JWKS. A fresh entry is served directly; a stale-but-good one is +/// served while a refresh runs off the request path (`spawn_jwks_refresh`), so a slow +/// issuer never stalls a request. Only a cold or negative entry blocks, under a per-URL +/// lock so a burst triggers one fetch; a failed fetch there caches a short-lived empty +/// entry that reads as "issuer unreachable", so an unreachable issuer is hit at most +/// once per `JWKS_NEGATIVE_TTL`. Fetches follow a schedule, never a per-request, +/// attacker-chosen `kid`. +async fn cached_jwks(url: &str) -> Result> { + if let Some(entry) = JWKS_CACHE.get(url) { + // Keys past JWKS_MAX_STALE are never served, even while revalidating: a stale re-serve + // bumps expires_at but keeps fetched_at, so a persistently failing refresh would + // otherwise serve revoked keys forever. Too-old keys fall through to the blocking + // refresh, which fails closed if the issuer is still down. + if entry.fetched_at.elapsed() < JWKS_MAX_STALE { + if entry.expires_at > Instant::now() { + return servable(entry); + } + // Stale but still holds good keys: serve them now and refresh off the request + // path, so a slow or hanging issuer adds no latency. Bump the entry first so the + // refresh window does not spawn a task per request. A negative (empty) entry + // falls through to the blocking refresh below. + if !entry.keys.is_empty() { + let served = Arc::new(JwksEntry { + keys: entry.keys.clone(), + expires_at: Instant::now() + JWKS_NEGATIVE_TTL, + fetched_at: entry.fetched_at, + }); + JWKS_CACHE.insert(url.to_string(), served.clone()); + spawn_jwks_refresh(url.to_string()); + return Ok(served); + } + } + } + // Cold or negative entry, nothing good to serve: block on a single-flight refresh. + // `JwksFetchLock::acquire` hands cold requests the same lock, so they share one fetch. + let fetch_lock = JwksFetchLock::acquire(url); + let _guard = fetch_lock.lock.lock().await; + // Another task may have refreshed while we waited for the lock (honour the age limit too, + // so a concurrent stale re-serve of too-old keys is not mistaken for a fresh entry). + if let Some(entry) = JWKS_CACHE.get(url) { + if entry.fetched_at.elapsed() < JWKS_MAX_STALE && entry.expires_at > Instant::now() { + return servable(entry); + } + } + match fetch_jwks(url).await { + Ok(keys) => { + let now = Instant::now(); + let entry = Arc::new(JwksEntry { + keys: Arc::new(keys), + expires_at: now + JWKS_TTL, + fetched_at: now, + }); + JWKS_CACHE.insert(url.to_string(), entry.clone()); + Ok(entry) + } + Err(e) => { + // Reached with no servable keys: either nothing cached, or keys too old to trust. + // Cache a short negative entry so the next requests do not each refetch, and + // surface the error, so an issuer that revoked a key or went down fails closed. + JWKS_CACHE.insert( + url.to_string(), + Arc::new(JwksEntry { + keys: Arc::new(HashMap::new()), + expires_at: Instant::now() + JWKS_NEGATIVE_TTL, + fetched_at: Instant::now(), + }), + ); + Err(e) + } + } +} + +/// The key a token's header selects from the workspace's JWKS, by `kid`, and the +/// algorithms it may verify. An unknown `kid` is refused against the cached set rather +/// than triggering a fetch, so varying `kid` cannot drive outbound requests; a +/// genuinely rotated-in key is picked up within `JWKS_TTL`. +pub async fn jwks_key_for(url: &str, token: &str) -> Result<(DecodingKey, Vec)> { + let header = jsonwebtoken::decode_header(token) + .map_err(|e| Error::NotAuthorized(format!("guest JWT refused: {e}")))?; + let kid = header.kid.ok_or_else(|| { + Error::NotAuthorized("guest JWT refused: no kid in the header".to_string()) + })?; + let entry = cached_jwks(url).await?; + let jwk = entry.keys.get(&kid).ok_or_else(|| { + Error::NotAuthorized(format!("guest JWT refused: kid {kid} is not in the JWKS")) + })?; + let algs = jwk_algorithms(jwk).ok_or_else(|| { + Error::NotAuthorized(format!("guest JWT refused: kid {kid} is not a signing key")) + })?; + let key = DecodingKey::from_jwk(jwk) + .map_err(|e| Error::internal_err(format!("unusable JWK {kid}: {e}")))?; + Ok((key, algs)) +} + +/// Verify `token` for `w_id` against whatever key the workspace configured. A PEM key +/// ignores `kid`; a JWKS selects by it. +pub async fn verify_for_workspace(db: &DB, w_id: &str, token: &str) -> Result { + // The admit check downstream refuses these anyway; refusing here keeps a deployment + // with no guests from parsing attacker-supplied JWTs at all, and names the reason in + // the log the caller writes. + if !crate::workspaces::instance_supports_guests() { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: {}", + crate::workspaces::GUESTS_UNAVAILABLE_MESSAGE + ))); + } + if token.len() > MAX_GUEST_JWT_LEN { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: token is longer than {MAX_GUEST_JWT_LEN} bytes" + ))); + } + let Some(source) = key_source(db, w_id).await? else { + return Err(Error::NotAuthorized(format!( + "guest JWT refused: workspace {w_id} has no guest JWT key" + ))); + }; + match source { + GuestJwtKeySource::Pem(pem) => { + let (key, algorithms) = decoding_key_from_pem(&pem)?; + verify(token, &key, algorithms, w_id) + } + GuestJwtKeySource::JwksUrl(url) => { + let (key, algorithms) = jwks_key_for(&url, token).await?; + verify(token, &key, &algorithms, w_id) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Serializes the tests that mutate the process-wide `ALLOW_PRIVATE_GUEST_JWKS_URLS`, so a + /// concurrent run cannot clear it out from under another (mirrors `ssrf.rs`'s test lock). + static TEST_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + fn jwk(v: serde_json::Value) -> Jwk { + serde_json::from_value(v).unwrap() + } + + #[test] + fn rsa_key_with_alg_pins_it() { + let k = jwk(serde_json::json!({"kty":"RSA","alg":"RS384","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), Some(vec![Algorithm::RS384])); + } + + #[test] + fn rsa_key_without_alg_takes_the_whole_family() { + // The bug this pins: an alg-less RSA key must not be forced to RS256, which + // would reject valid RS384/512 or PS* tokens. + let k = jwk(serde_json::json!({"kty":"RSA","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), Some(RSA_ALGORITHMS.to_vec())); + } + + #[test] + fn ec_key_takes_its_curve_algorithm() { + let k = jwk(serde_json::json!({"kty":"EC","crv":"P-256","x":"aa","y":"bb"})); + assert_eq!(jwk_algorithms(&k), Some(vec![Algorithm::ES256])); + } + + #[test] + fn symmetric_key_is_refused() { + let k = jwk(serde_json::json!({"kty":"oct","k":"c2VjcmV0"})); + assert_eq!(jwk_algorithms(&k), None); + } + + #[test] + fn a_key_naming_an_encryption_alg_is_refused() { + let k = jwk(serde_json::json!({"kty":"RSA","alg":"RSA-OAEP","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), None); + } + + #[test] + fn a_key_marked_for_encryption_is_refused() { + let k = jwk(serde_json::json!({"kty":"RSA","use":"enc","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), None); + } + + #[test] + fn a_mixed_use_jwks_keeps_only_the_signing_keys() { + // An encryption key (RSA-OAEP is not in jsonwebtoken's signing Algorithm enum) + // beside a signing key must not fail the whole set. The signing key carries real + // coordinates so it survives the material check parse_jwks_keys now applies. + let body = serde_json::json!({ + "keys": [ + {"kty":"RSA","alg":"RSA-OAEP","kid":"enc","use":"enc","n":"aa","e":"AQAB"}, + {"kty":"EC","crv":"P-256","kid":"sig","x":PUB1_X,"y":PUB1_Y} + ] + }) + .to_string(); + let keys = parse_jwks_keys(body.as_bytes()).expect("the signing key survives"); + assert!(keys.contains_key("sig")); + assert!(!keys.contains_key("enc")); + } + + #[test] + fn a_jwks_with_only_malformed_key_material_is_refused() { + // Metadata (kty/alg/use) is fine but `n` is not valid base64url, so the key is + // unusable. Since it is the only key, configuring this JWKS must fail at save time + // rather than persist a URL whose tokens all fail later. + let body = serde_json::json!({ + "keys": [{"kty":"RSA","kid":"k1","n":"not base64url!!","e":"AQAB"}] + }) + .to_string(); + assert!(parse_jwks_keys(body.as_bytes()).is_err()); + } + + #[test] + fn too_many_usable_keys_is_refused() { + // All keys are usable (real coordinates), so this trips the count cap, not the + // empty-set path. Small material, so it is the count that refuses them, not the bytes. + let keys: Vec<_> = (0..=JWKS_MAX_KEYS) + .map(|i| { + serde_json::json!({"kty":"EC","crv":"P-256","kid":format!("k{i}"),"x":PUB1_X,"y":PUB1_Y}) + }) + .collect(); + let body = serde_json::json!({ "keys": keys }).to_string(); + let err = parse_jwks_keys(body.as_bytes()).unwrap_err().to_string(); + assert!(err.contains("usable signing keys"), "{err}"); + } + + #[test] + fn keys_retaining_too_many_bytes_is_refused() { + // Under the key count cap, but the material of these usable RSA keys exceeds the byte + // cap. `from_jwk` decodes any-length `n`, so this is reachable without the count cap. + let big_n = "A".repeat(2000); + let keys: Vec<_> = (0..40) + .map(|i| serde_json::json!({"kty":"RSA","kid":format!("k{i}"),"n":big_n,"e":"AQAB"})) + .collect(); + let body = serde_json::json!({ "keys": keys }).to_string(); + let err = parse_jwks_keys(body.as_bytes()).unwrap_err().to_string(); + assert!(err.contains("retain more than"), "{err}"); + } + + #[tokio::test] + async fn an_overlong_jwks_url_is_refused() { + // The cache keys on the URL string, so an unbounded URL is refused before it is cached. + // Assert the length error specifically: a bogus URL would fail the fetch regardless. + let url = format!( + "https://issuer.example.com/{}", + "a".repeat(MAX_JWKS_URL_LEN) + ); + let err = fetch_jwks(&url).await.unwrap_err().to_string(); + assert!(err.contains("longer than"), "{err}"); + } + + #[test] + fn an_oversized_pem_is_refused() { + // A well-formed key body padded past the cap: refused for its length before decoding, + // so an oversized-but-valid key cannot be stored and reparsed on every request. + let big = format!( + "-----BEGIN PUBLIC KEY-----\n{}\n-----END PUBLIC KEY-----\n", + "A".repeat(MAX_GUEST_PEM_LEN) + ); + let err = decoding_key_from_pem(&big).err().unwrap().to_string(); + assert!(err.contains("longer than"), "{err}"); + // Whitespace padding must count: the endpoint stores the untrimmed value, so the cap is + // measured before trimming rather than on the small trimmed key it would otherwise see. + let padded = format!("{}{RSA_PUBLIC}", " ".repeat(MAX_GUEST_PEM_LEN)); + let err = decoding_key_from_pem(&padded).err().unwrap().to_string(); + assert!(err.contains("longer than"), "{err}"); + } + + #[test] + fn key_ops_without_verify_is_refused() { + let enc = jwk(serde_json::json!({"kty":"RSA","key_ops":["encrypt"],"n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&enc), None); + let ver = jwk(serde_json::json!({"kty":"RSA","key_ops":["verify"],"n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&ver), Some(RSA_ALGORITHMS.to_vec())); + } + + // PUB1's coordinates and its matching PKCS8 private key, for the JWKS-derived + // verification test. + const PUB1_X: &str = "zAfqyCh34iYOCW0vg4ejq_zzJlzLSZScjnVyPjLGTao"; + const PUB1_Y: &str = "RMKOIHOv8tWLnXf7-eMCodDnX038wCjD1sf9jVsf7oI"; + const PRIV1: &str = "-----BEGIN PRIVATE KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgu27S2DbSwUh8BmQb\n/i4/VhNdoXV7PJekhnoceMULYLihRANCAATMB+rIKHfiJg4JbS+Dh6Or/PMmXMtJ\nlJyOdXI+MsZNqkTCjiBzr/LVi513+/njAqHQ519N/MAow9bH/Y1bH+6C\n-----END PRIVATE KEY-----\n"; + + #[test] + fn a_jwks_key_verifies_a_real_token() { + // The one path the PEM tests do not cover: a key rebuilt from a JWK verifies a + // token signed by its private half, and the algorithms come from the JWK. + let jwk = jwk(serde_json::json!({ + "kty": "EC", "crv": "P-256", "kid": "k1", "x": PUB1_X, "y": PUB1_Y + })); + let algs = jwk_algorithms(&jwk).expect("EC signing key"); + assert_eq!(algs, vec![Algorithm::ES256]); + let key = jsonwebtoken::DecodingKey::from_jwk(&jwk).expect("usable JWK"); + let payload = serde_json::json!({ + "email": "g@example.com", + "workspace_id": "ws", + "app_path": "u/a/app", + "exp": jsonwebtoken::get_current_timestamp() + 600, + }); + let token = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &payload, + &jsonwebtoken::EncodingKey::from_ec_pem(PRIV1.as_bytes()).unwrap(), + ) + .unwrap(); + let out = verify(&token, &key, &algs, "ws").expect("verifies"); + assert_eq!(out.email, "g@example.com"); + // The workspace pin is part of verify. + assert!(verify(&token, &key, &algs, "other-ws").is_err()); + } + + #[test] + fn a_non_key_pem_is_rejected() { + assert!(decoding_key_from_pem( + "-----BEGIN CERTIFICATE-----\nnope\n-----END CERTIFICATE-----" + ) + .is_err()); + } + + #[test] + fn a_token_carrying_an_audience_still_verifies() { + // jsonwebtoken validates `aud` by default when the token has one, which would refuse + // every issuer that stamps its own audience. + let key = jsonwebtoken::DecodingKey::from_jwk(&jwk(serde_json::json!({ + "kty": "EC", "crv": "P-256", "kid": "k1", "x": PUB1_X, "y": PUB1_Y + }))) + .unwrap(); + let token = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &serde_json::json!({ + "email": "g@example.com", + "workspace_id": "ws", + "app_path": "u/a/app", + "aud": "https://issuer.example/api", + "exp": jsonwebtoken::get_current_timestamp() + 600, + }), + &jsonwebtoken::EncodingKey::from_ec_pem(PRIV1.as_bytes()).unwrap(), + ) + .unwrap(); + verify(&token, &key, &[Algorithm::ES256], "ws").expect("aud is not validated"); + } + + #[test] + fn an_8192_bit_rsa_key_verifies() { + // The largest modulus real issuers use; a crypto backend capped at 4096 bits rejects + // every token signed with it while the key itself still loads. + const PRIVATE: &str = include_str!("../tests/fixtures/guest_jwt_rsa8192_private.pem"); + const PUBLIC: &str = include_str!("../tests/fixtures/guest_jwt_rsa8192_public.pem"); + let (key, algs) = decoding_key_from_pem(PUBLIC).expect("8192-bit SPKI loads"); + let token = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::RS256), + &serde_json::json!({ + "email": "g@example.com", + "workspace_id": "ws", + "app_path": "u/a/app", + "exp": jsonwebtoken::get_current_timestamp() + 600, + }), + &jsonwebtoken::EncodingKey::from_rsa_pem(PRIVATE.as_bytes()).unwrap(), + ) + .unwrap(); + verify(&token, &key, algs, "ws").expect("8192-bit RSA signature verifies"); + } + + // A real RSA public key (SPKI). Its private counterpart is RSA_PKCS1_PRIVATE below. + const RSA_PUBLIC: &str = "-----BEGIN PUBLIC KEY-----\n\ +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx3J0fQcHp2ZlMI4rCVsY\n\ +tirATZPWyPD7exoYWPInhV5xjbY2Fe8IVFaZszQcQbCXZjBtFp2fj0tBTow8BeOy\n\ +X9LJPyKeho/j68FycuDVg7JCzG0TWtsnh/V23WkrlKIfmMqS3+YUyFavROTcAN1T\n\ +5BcFHLAi/4Q2qy0JjXBdZ8avelzZrQ/T67/Kcsoct/pvnEDT2YRsSbA7VMWaxWh8\n\ +MYJ7GNV/10YT2c5CBJGSLbyRSVWk2IwfnM9Cl9n/5NE6TkSetYQ2xlqKTONp5W43\n\ +UzW1NAeqKCxPQfN/ADjwW18nk2o7xj1kMF4rBlhsTm9ClE71nwi5NxsvMOdVxytZ\n\ +ZQIDAQAB\n\ +-----END PUBLIC KEY-----\n"; + + // A complete, valid PKCS#1 RSA *private* key (the counterpart of RSA_PUBLIC) with its + // armor relabelled `RSA PUBLIC KEY`. jsonwebtoken's from_rsa_pem accepts it under that + // label; the structural check refuses it (a 9-field RSAPrivateKey is neither an SPKI nor + // a 2-field RsaPublicKey). Complete on purpose: malformed DER would fail for the wrong + // reason and let a real bypass through unnoticed. + const RSA_PKCS1_PRIVATE_AS_PUBLIC: &str = "-----BEGIN RSA PUBLIC KEY-----\n\ +MIIEogIBAAKCAQEAx3J0fQcHp2ZlMI4rCVsYtirATZPWyPD7exoYWPInhV5xjbY2\n\ +Fe8IVFaZszQcQbCXZjBtFp2fj0tBTow8BeOyX9LJPyKeho/j68FycuDVg7JCzG0T\n\ +Wtsnh/V23WkrlKIfmMqS3+YUyFavROTcAN1T5BcFHLAi/4Q2qy0JjXBdZ8avelzZ\n\ +rQ/T67/Kcsoct/pvnEDT2YRsSbA7VMWaxWh8MYJ7GNV/10YT2c5CBJGSLbyRSVWk\n\ +2IwfnM9Cl9n/5NE6TkSetYQ2xlqKTONp5W43UzW1NAeqKCxPQfN/ADjwW18nk2o7\n\ +xj1kMF4rBlhsTm9ClE71nwi5NxsvMOdVxytZZQIDAQABAoIBAD+IbaQQM7d3Dj/X\n\ +4cyyqJ4K40QzFmXfIfTWXLAkv0MkUR7XzsXQ5YHcLkzgCipAwxGp1m4wWs4OJmkL\n\ +kek8XatZnYLPl9j8iBmm/zqp9Unk5JNzIYm9KwwLvMgOAvRvaopE6WGKTM9+kYls\n\ +L8rUti7/yECZuSRU7Qc9KwBTrWVrXK+RBtBqZYQXb92BFxq0N3Qp+utLNdFcO5sW\n\ +7d8gKp3ipQt5z9ZAB2pYMw7ZTzonF4C7HdyrbYXztvYrxuw1imMkQ9iFFhdn3/76\n\ +qFR7XwaFrld8DECGaH/652kV6zaSQijbBTeXF4zsgwXY4BHMVmZKaXH5unw8Gbmo\n\ +WCoLbLcCgYEA4vTo5kCiKXnlBp3W1Zpg4cml6Wzo0UDldF4kkuXQxhdQA38c0ise\n\ +Cocf6qyAqz1L2TxQ/9WCL2oIP1AY9XqnQ0cJtYIosGWORz4tPe67M8inB/GBotFI\n\ +pmQNVSIjqbgKVi0x+UzmFjitINFPf461lDdJTwhsv9TQRbHrXErvON8CgYEA4PhV\n\ +GYMJu46tqFVtD/koWAQRLmeaZXhxP5lMSmQjdYCa3ys5lccvTlzoF9K8immMQkIx\n\ +gyOazmEtFnK4IXmEY1wg2NIHuJM7/maoM2rozbjXBxsYM7Xw2QX7BHXqG6Ia/Bij\n\ +ZaRJdumCVRJv7OshQTGuqDIzd3l5WEqg11XYgjsCgYAP3v6Wc3ijm+GXN9x5LYWO\n\ +5JIUo8gYMgiZvaejGi0iXSj8RZxXWiqMo+xodc29q9itBVnIuj6TYD/ZZZmJOR2P\n\ +R9128vYzd7aeZsu1JAe1VFfR52KgZzBEaoTAKlYCHVujsR9ohqckcKwyulBr5Cfw\n\ +iHk47KbmN1SlOw7xclAOUwKBgAuxHEsdIk5bFe9fsTFZU51vaK0uuTl4zvntL6fW\n\ +GHms21+p0W5VUcIS1gUW8LGI1r9CzWvxV8RODJfUEnm65QR870AVek0/aajJEQjL\n\ +D5pRdutpnxJg7El7JBaRQj95Z0mexi8sIJ1LeXiOYr6/YZUPzfHz2fTlnUbXahCG\n\ +55+tAoGAI811NTb7kuuIPYuj4raDW88QVNX2xB3+p9lXGolB4jPgsUEjgSvLgH9S\n\ +Q/LwEBiCYVyii8MvWsIZpHvSGyOoty2p19/CAvrAOfpEVlnXQeiX+mh09p1mQbfM\n\ +y9rTR828ADcaZ63Ej1oL4GcqmGhODxCLy1YKKcy0FHzChqPMV6g=\n\ +-----END RSA PUBLIC KEY-----\n"; + + #[test] + fn a_private_pem_is_refused() { + // A verification key must be public and must never be stored otherwise: it is served + // back through the settings response. jsonwebtoken keys the public/private split off + // the PEM label, so private DER relabelled with a public armor slips a label check; + // only parsing the DER as a public-key structure (SPKI or PKCS#1 RSA public) refuses + // it. A real public key still parses, so the guard is not vacuous. + assert!(decoding_key_from_pem(RSA_PUBLIC).is_ok()); + // The same key with its body on one line (not 64-column wrapped): jsonwebtoken accepts + // any wrapping, so the guard must too rather than lean on the strict RFC 7468 decoder. + let body: String = RSA_PUBLIC + .lines() + .filter(|l| !l.starts_with("-----")) + .collect(); + let one_line = format!("-----BEGIN PUBLIC KEY-----\n{body}\n-----END PUBLIC KEY-----\n"); + assert!(decoding_key_from_pem(&one_line).is_ok()); + + assert!(decoding_key_from_pem(PRIV1).is_err()); + // The same PKCS#8 EC private key, relabelled `PUBLIC KEY`. + assert!(decoding_key_from_pem(&PRIV1.replace("PRIVATE KEY", "PUBLIC KEY")).is_err()); + assert!(decoding_key_from_pem(RSA_PKCS1_PRIVATE_AS_PUBLIC).is_err()); + } + + #[tokio::test] + async fn a_concurrent_cold_burst_makes_one_jwks_fetch() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::AsyncWriteExt; + let _env = TEST_ENV_LOCK.lock().await; + // The stub listens on loopback, which SSRF validation refuses without this. + unsafe { std::env::set_var("ALLOW_PRIVATE_GUEST_JWKS_URLS", "true") }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let body = format!( + r#"{{"keys":[{{"kty":"EC","crv":"P-256","kid":"k1","x":"{PUB1_X}","y":"{PUB1_Y}"}}]}}"# + ); + let hits = std::sync::Arc::new(AtomicUsize::new(0)); + let hits_srv = hits.clone(); + tokio::spawn(async move { + loop { + let (mut sock, _) = listener.accept().await.unwrap(); + hits_srv.fetch_add(1, Ordering::SeqCst); + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + // Delay so the other callers pile onto the single-flight lock before the + // leader's fetch returns. + tokio::time::sleep(Duration::from_millis(150)).await; + let _ = sock.write_all(resp.as_bytes()).await; + let _ = sock.shutdown().await; + } + }); + let url = format!("http://127.0.0.1:{}/jwks.json", addr.port()); + let mut handles = Vec::new(); + for _ in 0..10 { + let u = url.clone(); + handles.push(tokio::spawn(async move { + cached_jwks(&u).await.map(|e| e.keys.len()) + })); + } + for h in handles { + assert_eq!( + h.await.unwrap().unwrap(), + 1, + "each caller resolves the one key" + ); + } + assert_eq!( + hits.load(Ordering::SeqCst), + 1, + "single-flight: a concurrent cold burst makes one fetch" + ); + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + } + + #[tokio::test] + async fn jwks_fetch_locks_are_shared_and_self_cleaning() { + // The registry is a plain map, not a capacity-bounded cache: a cache could evict a + // lock mid-fetch, letting a later request for that URL start a duplicate fetch. Pin + // both halves of what keeps single-flight intact under many distinct URLs: the same + // URL hands back one shared lock, and the entry is removed once its last holder drops + // (so nothing evicts an in-flight lock and the map stays bounded by fetches in flight). + let url = "https://example.test/jwks-lock-probe.json"; + { + let a = JwksFetchLock::acquire(url); + let b = JwksFetchLock::acquire(url); + assert!(Arc::ptr_eq(&a.lock, &b.lock), "one lock per URL"); + assert!(JWKS_FETCH_LOCKS.lock().unwrap().contains_key(url)); + } + assert!( + !JWKS_FETCH_LOCKS.lock().unwrap().contains_key(url), + "the lock is dropped once idle" + ); + } + + #[tokio::test] + async fn stale_jwks_keys_stop_being_served_past_the_grace_window() { + let _env = TEST_ENV_LOCK.lock().await; + unsafe { std::env::set_var("ALLOW_PRIVATE_GUEST_JWKS_URLS", "true") }; + // A dead loopback port, so every refresh fails (connection refused). + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let jwk = jwk(serde_json::json!( + {"kty":"EC","crv":"P-256","kid":"k1","x":PUB1_X,"y":PUB1_Y} + )); + let keys: HashMap = [("k1".to_string(), jwk)].into_iter().collect(); + let stale = Instant::now().checked_sub(Duration::from_secs(1)).unwrap(); + + // Within the grace window, stale keys are still served while a (failing) refresh runs. + let within = format!("http://127.0.0.1:{port}/within"); + JWKS_CACHE.insert( + within.clone(), + Arc::new(JwksEntry { + keys: Arc::new(keys.clone()), + expires_at: stale, + fetched_at: Instant::now(), + }), + ); + assert!( + cached_jwks(&within).await.is_ok(), + "stale keys within the grace window are still served" + ); + + // Past the grace window, the keys are not served: the failing refresh fails closed. + let beyond = format!("http://127.0.0.1:{port}/beyond"); + JWKS_CACHE.insert( + beyond.clone(), + Arc::new(JwksEntry { + keys: Arc::new(keys), + expires_at: stale, + fetched_at: Instant::now() + .checked_sub(JWKS_MAX_STALE + Duration::from_secs(1)) + .unwrap(), + }), + ); + assert!( + cached_jwks(&beyond).await.is_err(), + "keys past the grace window fail closed once the refresh fails" + ); + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + } + + #[tokio::test] + async fn a_plaintext_http_jwks_url_is_refused_by_default() { + let _env = TEST_ENV_LOCK.lock().await; + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + // The JWKS supplies the keys that authenticate guest JWTs; without the operator opt-in, + // a plaintext URL (which an on-path attacker could replace) is refused for its scheme. + assert!(matches!( + crate::ssrf::validate_guest_jwks_url("http://issuer.example.com/jwks.json").await, + Err(crate::ssrf::SsrfValidationError::HttpsRequired) + )); + } + + #[tokio::test] + async fn the_instance_issuer_bypasses_the_https_and_private_restriction() { + let _env = TEST_ENV_LOCK.lock().await; + unsafe { std::env::remove_var("ALLOW_PRIVATE_GUEST_JWKS_URLS") }; + // The instance issuer is operator-trusted (it also backs jwt_ext_), so an http/private + // URL is not refused for its scheme: it reaches the connect and fails there (dead port), + // not at validation. A different URL is not the instance issuer and is still refused. + let instance = "http://127.0.0.1:1/jwks.json"; + unsafe { std::env::set_var("JWT_EXT_JWKS_URL", instance) }; + let trusted = fetch_jwks(instance).await.err().unwrap().to_string(); + let other = fetch_jwks("http://127.0.0.1:1/other.json") + .await + .err() + .unwrap() + .to_string(); + unsafe { std::env::remove_var("JWT_EXT_JWKS_URL") }; + assert!( + !trusted.contains("not allowed") && !trusted.contains("must use https"), + "instance issuer skips validation: {trusted}" + ); + assert!( + other.contains("not allowed") || other.contains("https"), + "a non-instance http url is still refused: {other}" + ); + } +} diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 7b1ed407ab..559eba3bf9 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -94,6 +94,21 @@ pub async fn load_indexer_config(db: &DB) -> error::Result i64 { + let retention = crate::service_log_retention_secs(); + if max_index_time_window_secs > 0 { + std::cmp::min(max_index_time_window_secs, retention) + } else { + retention + } +} + pub fn get_env_var(env_var: &str) -> Option { match std::env::var(env_var).map(|x| x.parse()) { Ok(Ok(i)) => Some(i), @@ -136,3 +151,60 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings { settings } + +#[cfg(test)] +mod tests { + use super::*; + + // One test rather than several: both halves share the process-wide retention, and the + // setter half writes it, which parallel tests would race. + #[test] + fn retention_rejects_unusable_values_and_the_index_window_clamps_to_it() { + use crate::{ + service_log_retention_secs, set_service_log_retention_secs, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + }; + + // See `set_service_log_retention_secs` for why the two unusable directions land apart: + // too large keeps the intent by capping, non-positive cannot and falls back. + let rejected: Vec = [0, -1, i64::MIN] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + let capped: Vec = [i64::MAX, 60 * 60 * 24 * 365 * 101] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + + set_service_log_retention_secs(60 * 60 * 24 * 3); + let retention = service_log_retention_secs(); + let windows = [ + // `0` disables the extra shrinking rather than lifting the ceiling — the trap that + // makes an unset setting look unbounded. + service_log_index_window_secs(0), + // Retention is the ceiling: the index cannot reach lines whose `log_file` row is gone. + service_log_index_window_secs(retention * 2), + service_log_index_window_secs(60), + ]; + set_service_log_retention_secs(DEFAULT_SERVICE_LOG_RETENTION_SECS); + + assert_eq!( + rejected, + vec![DEFAULT_SERVICE_LOG_RETENTION_SECS; 3], + "a value that would expire everything must fall back to the default" + ); + assert_eq!( + capped, + vec![60 * 60 * 24 * 365 * 100; 2], + "an oversized value must cap, not shorten retention to the default" + ); + assert_eq!(retention, 60 * 60 * 24 * 3); + assert_eq!(windows, [retention, retention, 60]); + } +} diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 7ea6dbe5ae..14ef869b37 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -264,6 +264,8 @@ pub struct GlobalSettings { pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ai_sessions_instance_storage_fallback: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] @@ -829,6 +831,8 @@ pub struct CustomInstanceDbLogs { pub replication_user: String, #[serde(skip_serializing_if = "Option::is_none")] pub replication_user_error: Option, + #[serde(skip_serializing_if = "String::is_empty")] + pub user_connect: String, } // --------------------------------------------------------------------------- @@ -916,6 +920,14 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Object store this group's dependency cache uses instead of the instance one. Same shape + /// as the instance `object_store_cache_config`. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "instance_config_schema", + schemars(schema_with = "opaque_json_schema") + )] + pub object_store_cache_config: Option, /// Catch-all for fields not yet covered by typed fields. #[serde(flatten)] @@ -1280,8 +1292,9 @@ pub fn diff_worker_configs( ConfigsDiff { upserts, deletes } } -/// Declaratively replace the global settings, rejecting a `github_app_webhook_base_url` -/// the API would reject. +/// Declaratively replace the global settings, rejecting a +/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the +/// API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1330,6 +1343,27 @@ pub async fn sync_global_settings_declarative( } } + let banner_key = crate::global_settings::INSTANCE_BANNER_SETTING; + match desired.get(banner_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(banner) => crate::global_settings::validate_instance_banner(banner) + // The validator's messages name the offending field and its expected type, + // never the submitted value, so they are safe to surface here. + .map_err(|e| anyhow::anyhow!("{banner_key}: {e}"))?, + } + + // An origin list that cannot be parsed is dropped at boot, leaving the + // empty default — which is no restriction at all. Rejecting it here is what + // keeps a typo in a ConfigMap from silently widening CORS instance-wide. + let origins_key = crate::global_settings::HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING; + crate::global_settings::parse_allowed_origins_setting(desired.get(origins_key)) + .map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?; + + let max_expiration_key = crate::global_settings::MAX_TOKEN_EXPIRATION_DAYS_SETTING; + crate::global_settings::parse_max_token_expiration_days(desired.get(max_expiration_key)) + .map_err(|e| anyhow::anyhow!("{max_expiration_key}: {e}"))?; + let diff = diff_global_settings(current, desired, ApplyMode::Replace); apply_settings_diff(db, &diff).await?; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5541615838..6f3248f3fb 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -478,6 +478,12 @@ pub static WORKER_INTERNAL_SERVER_INLINE_UTILS: OnceCell error::Result<()> { sqlx::query!( "DELETE FROM dispatch_event WHERE producer_job_id = ANY($1)", @@ -485,12 +491,55 @@ pub async fn delete_jobs(conn: &mut sqlx::PgConnection, ids: &[uuid::Uuid]) -> e ) .execute(&mut *conn) .await?; - sqlx::query!( - "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)", + let mut conversation_ids: Vec = sqlx::query_scalar!( + "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", ids ) - .execute(&mut *conn) + .fetch_all(&mut *conn) .await?; + conversation_ids.sort_unstable(); + conversation_ids.dedup(); + if !conversation_ids.is_empty() { + // A conversation is a view over its messages: once the last one goes with its job, + // the row and the agent's memory for it are all that is left, and nothing else + // collects them — `ai_agent_memory` carries no job id for retention to match on. + // Two statements rather than one CTE: a data-modifying CTE reads the snapshot from + // before the delete above, so every conversation would still look non-empty. + // Two calls each deleting one of a conversation's last messages would each still see + // the other's row — uncommitted deletes are invisible across transactions — so + // neither would collect it and nothing would try again. Taking the conversation row + // first serialises them: the second reads the first's delete and finds it empty. + sqlx::query_scalar!( + "SELECT id FROM flow_conversation WHERE id = ANY($1) ORDER BY id FOR UPDATE", + &conversation_ids + ) + .fetch_all(&mut *conn) + .await?; + // Memory first, since it reads the conversation row for its workspace. + sqlx::query!( + "DELETE FROM ai_agent_memory a + USING flow_conversation c + WHERE c.id = ANY($1) + AND a.conversation_id = c.id + AND a.workspace_id = c.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "DELETE FROM flow_conversation c + WHERE c.id = ANY($1) + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids + ) + .execute(&mut *conn) + .await?; + } sqlx::query!("DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", ids) .execute(&mut *conn) .await?; diff --git a/backend/windmill-common/src/jwt.rs b/backend/windmill-common/src/jwt.rs index cc2f1102bf..76df13b518 100644 --- a/backend/windmill-common/src/jwt.rs +++ b/backend/windmill-common/src/jwt.rs @@ -3,7 +3,6 @@ use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use hmac::{Hmac, Mac}; use serde::{de::DeserializeOwned, Serialize}; use sha2::Sha256; -use std::collections::HashSet; lazy_static::lazy_static! { pub static ref JWT_SECRET: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); @@ -44,18 +43,8 @@ pub async fn decode_with_internal_secret(token: &str) -> er } pub fn decode_without_verify(token: &str) -> anyhow::Result { - // Create a validation that skips all checks - let mut validation = jsonwebtoken::Validation::default(); - validation.insecure_disable_signature_validation(); - validation.validate_exp = false; - validation.validate_nbf = false; - validation.required_spec_claims = HashSet::new(); - - // Use an empty key since we're not verifying - let key = jsonwebtoken::DecodingKey::from_secret(&[]); - - // Decode the token - let token_data = jsonwebtoken::decode::(token, &key, &validation)?; + // No signature, expiry, nbf or required-claim checks: the caller wants the raw claims. + let token_data = jsonwebtoken::dangerous::insecure_decode::(token)?; Ok(token_data.claims) } @@ -74,3 +63,36 @@ pub async fn generate_signature(header_and_payload: &str) -> anyhow::Result Option { + // `alg` is a `KeyAlgorithm` (signing and encryption algorithms alike); only the + // signing subset maps onto `Algorithm`. + jwk.common + .key_algorithm + .and_then(|alg| alg.to_string().parse::().ok()) +} + +#[cfg(test)] +mod tests { + use super::jwk_algorithm; + use jsonwebtoken::{jwk::Jwk, Algorithm}; + + #[test] + fn jwk_algorithm_reads_alg_when_present() { + let with: Jwk = + serde_json::from_value(serde_json::json!({"kty":"RSA","alg":"RS256","n":"aa","e":"AQAB"})) + .unwrap(); + assert_eq!(jwk_algorithm(&with), Some(Algorithm::RS256)); + let without: Jwk = + serde_json::from_value(serde_json::json!({"kty":"RSA","n":"aa","e":"AQAB"})).unwrap(); + assert_eq!(jwk_algorithm(&without), None); + let encryption: Jwk = serde_json::from_value( + serde_json::json!({"kty":"RSA","alg":"RSA-OAEP","n":"aa","e":"AQAB"}), + ) + .unwrap(); + assert_eq!(jwk_algorithm(&encryption), None); + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index cf0be2bbd8..3e74d8e17f 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -37,6 +37,10 @@ pub mod bench; pub mod cache; pub mod client; pub mod data_metrics; +pub mod datatable_roles; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod datatable_roles_ee; +pub mod datatable_roles_oss; pub mod db; #[cfg(all(feature = "enterprise", feature = "private"))] mod db_entra_ee; @@ -67,6 +71,7 @@ pub mod flow_status; pub mod flows; pub mod folders; pub mod global_settings; +pub mod guest_jwt; pub mod indexer; pub mod instance_config; pub mod job_metrics; @@ -110,6 +115,7 @@ pub use pipeline_advanced_ee as pipeline_advanced; pub use pipeline_advanced_oss as pipeline_advanced; pub mod query_builders; pub mod queue; +pub mod queue_metrics; pub mod result_stream; pub mod runnable_settings; pub mod schedule; @@ -147,9 +153,94 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; -pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans +pub const DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = 60 * 60 * 24 * 30; pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; +/// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower +/// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` +/// the cleanup queries build overflows Postgres' microsecond field. +const MAX_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; + +/// Clamp a configured retention window, in seconds, to one a cutoff can be built from. +/// +/// Shared by the retention windows that have no "keep forever" spelling, so that an unusable +/// value can never reach a cutoff. The two unusable directions are not the same mistake and must +/// not share a landing point: too large still says "keep these for a very long time", so it is +/// capped and the intent survives, whereas falling back would delete data the operator meant to +/// keep. A non-positive value has no such reading — every cutoff is `now - retention`, so it +/// lands at or after `now` and the next sweep expires the entire history. `0` is both what an +/// operator types by analogy with job retention, where it does mean keep forever, and what the +/// settings UI writes into a field that was merely focused, so it falls back to the default. +fn clamp_retention_secs(configured: i64, default: i64, what: &str) -> i64 { + if configured > MAX_RETENTION_SECS { + tracing::warn!( + "{what} retention of {configured}s exceeds the maximum of {MAX_RETENTION_SECS}s, \ + capping it there" + ); + MAX_RETENTION_SECS + } else if configured >= 1 { + configured + } else { + tracing::warn!( + "{what} retention of {configured}s would expire the entire history, \ + falling back to the default of {default}s" + ); + default + } +} + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`]. Expiry reaches every copy of a log line: +/// the row, the file on disk, and the object-storage object. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + "service log", + ); + SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// Apply a configured OTEL trace retention, in seconds. +/// +/// The only way into [`OTEL_TRACES_RETENTION_SECS`]. +pub fn set_otel_traces_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_OTEL_TRACES_RETENTION_SECS, + "otel traces", + ); + OTEL_TRACES_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long an HTTP request tracing span stays in `otel_traces`, in seconds. +/// +/// Spans are keyed by the job they were captured for and read back by the job detail view, so +/// this is the outer bound on how far back that view can show a job's HTTP requests. It is +/// independent of job retention: a span can outlive its job, or be swept while the job remains. +pub fn otel_traces_retention_secs() -> i64 { + OTEL_TRACES_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + +/// How long a service log line stays retrievable, in seconds. +/// +/// The outer bound on everything service-log: the `log_file` rows, the raw files in object +/// storage, the columnar store queried by retrieval, and — through +/// [`indexer::service_log_index_window_secs`] — the search index. +pub fn service_log_retention_secs() -> i64 { + SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + +/// How long a shared AI session artifact stays viewable, in seconds, counted from the last time +/// its author shared it. Read by both the API, which stops serving an expired share, and the +/// monitor, which deletes it — so both must agree, which is why they share this one reader. +pub fn ai_shared_artifact_retention_secs() -> i64 { + *AI_SHARED_ARTIFACT_RETENTION_SECS +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -203,14 +294,16 @@ pub fn check_on_behalf_of_preservation( None } -/// Resolves the identity to store when creating/updating a flow or script. +/// Resolves the identity to store when creating/updating a flow, script or app. /// -/// The permissioned_as is the only stored identity — it decides what the job may access, -/// and the address is derived from it at read time — so the two can never name different -/// accounts. Callers may supply either: a bare email (every client written before the -/// principal existed) is resolved to the principal it names, and an email that names -/// nobody is rejected rather than recorded, since it could only produce a runnable that -/// cannot authenticate. +/// The permissioned_as is the identity: it decides what the job may access, and the address is +/// a function of it, so the two can never name different accounts. For a script or flow the +/// address is derived at read time; an app still stores it, as a compatibility copy written +/// through from the principal on every save and returned verbatim by the app reads (see +/// `docs/app-policy-email-removal.md`). Callers may supply either: a bare email (every client +/// written before the principal existed) is resolved to the principal it names, and an email +/// that names nobody is rejected rather than recorded, since it could only produce a runnable +/// that cannot authenticate. /// /// Returns `None` when the runnable has no on-behalf-of identity, and the caller's own /// identity when they are not allowed to preserve someone else's. @@ -218,6 +311,18 @@ pub fn check_on_behalf_of_preservation( /// Resolves through the non-RLS pool and authorizes nothing itself — `authed` decides only /// whether preservation is allowed, and its role flags are not re-checked against `w_id`. /// Callers must already be authorized for the workspace they pass. +/// +/// Known, accepted race. The lookup runs on the pool, outside the caller's write transaction, so +/// an account renamed or removed between the two has its sweep run before the write is visible, +/// and the write stores the old principal. The runnable then fails to authenticate until it is +/// deployed with a current identity, with two exceptions: an app naming an external superadmin +/// keeps running as that account through its stored address, and if the freed username is later +/// given to another account, the stale principal binds to that account and runs as it. Every +/// caller shares this (scripts, flows and apps, address-only inputs included), and it needs a +/// rename or removal of the exact account inside the lookup-to-commit gap. Closing it means +/// serializing every identity write against every identity mutation, across all runnable kinds +/// (a `usr` row lock in each write, with each sweep ordered after the account change), which no +/// single caller can do on its own; it is left open deliberately. pub async fn resolve_on_behalf_of( on_behalf_of_email: Option<&str>, on_behalf_of: Option<&str>, @@ -375,6 +480,23 @@ lazy_static::lazy_static! { /// workspace configured before its override could be read. pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false); pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0); + /// Private on purpose: [`set_service_log_retention_secs`] is the only writer, so a value that + /// would expire every service log cannot reach a cutoff. Read it with + /// [`service_log_retention_secs`]. + static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); + /// Private on purpose, same as [`SERVICE_LOG_RETENTION_SECS`]: + /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the + /// only reader. + static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); + /// Read it with [`ai_shared_artifact_retention_secs`]. + static ref AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = clamp_retention_secs( + std::env::var("AI_SHARED_ARTIFACT_RETENTION_SECS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS), + DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS, + "AI shared artifact", + ); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); @@ -902,6 +1024,20 @@ impl Future for TokioPgConnection { } } +impl TokioPgConnection { + /// Drive the connection and hand back what the server sends outside of a query's response — + /// notices above all, which driving it as a future silently discards. + pub fn poll_message( + &mut self, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll>> { + match self { + TokioPgConnection::Tls(conn) => conn.poll_message(cx), + TokioPgConnection::NoTls(conn) => conn.poll_message(cx), + } + } +} + impl PgDatabase { /// The role the connection logs in as, whichever way it authenticates. pub fn login_name(&self) -> &str { @@ -1338,8 +1474,163 @@ pub fn validate_dbname(dbname: &str) -> error::Result<()> { Ok(()) } +/// Lock the instance databases among `names` until `tx` ends, in a stable order. Taken by every +/// settings save naming an instance database, and by [`drop_unused_instance_database`]: a save +/// cannot start using a database between that drop's check that nothing does and the drop. +pub async fn lock_instance_databases<'a>( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + names: impl IntoIterator, +) -> error::Result<()> { + let names: std::collections::BTreeSet<&str> = names.into_iter().collect(); + for name in names { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('instance_database:' || $1))") + .bind(name) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +/// Drop instance database `dbname`, which a request just created, unless a workspace names it as a +/// data table or a ducklake catalog — then it is kept, and who keeps it is what comes back. A +/// database is registered on the instance as soon as it is created, so a superadmin can point a +/// workspace at it before the request that created it gives up on it. +/// +/// Authorization: none. Callers MUST pass only a database the same request created and has not +/// handed to anything yet. +/// +/// The lock, the check and the drop share one connection: a second one taken from the pool while +/// the first is held could wait forever on a small pool. That connection is closed rather than +/// returned, since a session lock outlives the future holding it: a cancellation between taking +/// the lock and releasing it would otherwise hand a locked session back to the pool, where every +/// later settings save waits on it. Closing on drop covers the cancellation and still counts the +/// connection against the pool, which detaching it would not. +pub async fn drop_unused_instance_database(db: &DB, dbname: &str) -> error::Result { + let mut conn = db.acquire().await?; + conn.close_on_drop(); + let key = format!("instance_database:{dbname}"); + // A save blocked on this lock is about to name the database, but it may also roll back — a + // later validation of its own, a superadmin check, a cancelled request. So it is let through + // and what it committed is read, rather than taken as a user: trusting it would strand the + // database, whose name then blocks every retry. A fresh waiter can always arrive, so after a + // few rounds the database is kept instead; the name is then a superadmin's to drop from + // instance settings, since a retry fails on the name before reaching this cleanup. + let mut rounds = 0; + let dropped = loop { + if let Err(e) = sqlx::query("SELECT pg_advisory_lock(hashtext($1))") + .bind(&key) + .execute(&mut *conn) + .await + { + break Err(e.into()); + } + match drop_if_unused_on(&mut conn, dbname).await { + Ok(Cleanup::Waiter(_)) if rounds < 2 => { + rounds += 1; + // Releasing hands the lock to the waiter; re-taking it above then waits for that + // transaction to end, so the next round reads what it actually committed. + if let Err(e) = unlock_instance_database(&mut conn, &key).await { + break Err(e); + } + } + Ok(outcome) => break Ok(outcome), + Err(e) => break Err(e), + } + }; + // Dropping closes the connection, and the server releases the lock with the session, so + // nothing here depends on an unlock landing. + dropped +} + +async fn unlock_instance_database(conn: &mut sqlx::PgConnection, key: &str) -> error::Result<()> { + sqlx::query("SELECT pg_advisory_unlock(hashtext($1))") + .bind(key) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// What [`drop_unused_instance_database`] did, and for whom when it kept the database. +pub enum Cleanup { + Dropped, + /// Workspaces that name the database, as [`instance_database_users`] reports them. + InUse(Vec), + /// A transaction still blocked on the lock after the rounds above, so what it will commit + /// stays unknown, named as its `pid`. + Waiter(i32), +} + +async fn drop_if_unused_on(conn: &mut sqlx::PgConnection, dbname: &str) -> error::Result { + let users = instance_database_users(conn, dbname).await?; + if !users.is_empty() { + return Ok(Cleanup::InUse(users)); + } + // A settings save naming this database takes the same lock, so one waiting on it would read + // its own users after the drop and commit a reference to nothing. + if let Some(waiter) = waiting_for_instance_database(conn, dbname).await? { + return Ok(Cleanup::Waiter(waiter)); + } + drop_custom_instance_database_on(conn, dbname).await?; + Ok(Cleanup::Dropped) +} + +/// The `pid` of a transaction blocked on `dbname`'s instance-database lock. The lock is +/// taken by key, so `pg_locks` reports it split across `classid` and `objid`, and it lists every +/// database on the cluster — another Windmill on the same one holds its own locks under the same +/// key. +async fn waiting_for_instance_database( + conn: &mut sqlx::PgConnection, + dbname: &str, +) -> error::Result> { + let pid: Option = sqlx::query_scalar( + "SELECT l.pid FROM pg_locks l + WHERE l.locktype = 'advisory' AND NOT l.granted + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND l.classid = ((hashtext('instance_database:' || $1)::bigint >> 32) & 4294967295)::oid + AND l.objid = (hashtext('instance_database:' || $1)::bigint & 4294967295)::oid + LIMIT 1", + ) + .bind(dbname) + .fetch_optional(&mut *conn) + .await?; + Ok(pid) +} + +/// The workspaces naming instance database `dbname` as a data table or a ducklake catalog. Taken +/// under [`lock_instance_databases`] for `dbname`, the answer holds until that lock is released. +/// +/// Authorization: none, and it reads every workspace's settings. Callers MUST pass only a database +/// their own request created, and may name the workspaces returned only to a caller allowed to +/// create or drop instance databases. +pub async fn instance_database_users( + conn: &mut sqlx::PgConnection, + dbname: &str, +) -> error::Result> { + Ok(sqlx::query_scalar( + "SELECT DISTINCT ws.workspace_id FROM workspace_settings ws + WHERE EXISTS (SELECT 1 FROM jsonb_each(CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt + WHERE dt.value->'database'->>'resource_type' = 'instance' + AND dt.value->'database'->>'resource_path' = $1) + OR EXISTS (SELECT 1 FROM jsonb_each(CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object' + THEN ws.ducklake->'ducklakes' ELSE '{}'::jsonb END) dl + WHERE dl.value->'catalog'->>'resource_type' = 'instance' + AND dl.value->'catalog'->>'resource_path' = $1)", + ) + .bind(dbname) + .fetch_all(conn) + .await?) +} + /// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { + drop_custom_instance_database_on(&mut *db.acquire().await?, dbname).await +} + +async fn drop_custom_instance_database_on( + conn: &mut sqlx::PgConnection, + dbname: &str, +) -> error::Result<()> { let dbname = dbname.trim(); validate_dbname(dbname)?; @@ -1354,7 +1645,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", dbname ) - .fetch_one(db) + .fetch_one(&mut *conn) .await? .unwrap_or(false); @@ -1365,7 +1656,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()", dbname.replace('\'', "''") )) - .execute(db) + .execute(&mut *conn) .await { tracing::warn!("Failed to terminate connections to '{}': {}", dbname, e); @@ -1374,7 +1665,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu // Drop the database // SAFETY: `dbname` has been validated via validate_dbname() before reaching this point. sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) - .execute(db) + .execute(&mut *conn) .await .map_err(|e| { error::Error::internal_err(format!("Failed to drop database '{}': {}", dbname, e)) @@ -1390,12 +1681,47 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, dbname ) - .execute(db) + .execute(&mut *conn) .await?; Ok(()) } +/// What `custom_instance_user` holds on an instance database. +/// +/// `WITH GRANT OPTION` throughout: this is the connection every data table resolves to as `admin`, +/// and it is the one that hands privileges to data table roles. Postgres refuses to let a role pass +/// on a privilege it does not itself hold with grant option, so without these an admin could own +/// the database and still be unable to grant `SELECT` on it to `analytics`. +pub(crate) fn instance_db_grants(dbname: &str) -> String { + format!( + "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION; + GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION; + DO $$ BEGIN + IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN + GRANT USAGE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION; + GRANT CREATE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user; + END IF; + END $$;" + ) +} + +/// Re-apply [`instance_db_grants`] to an instance database provisioned before data table roles +/// existed, whose grants carry no grant option. Connects as the instance's own Postgres user — +/// the database and `public` schema owner — since only it can hand out an option it holds. +/// +/// Authorization: reaches an instance database with the server's own credentials and checks +/// nothing. Callers MUST have authorized administration of `dbname` — superadmin, or an admin of +/// the workspace governing a data table on it. +pub async fn ensure_instance_db_grant_options_unchecked( + db: &DB, + dbname: &str, +) -> error::Result<()> { + crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, dbname).await +} + /// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings. /// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). pub async fn create_custom_instance_database( @@ -1429,23 +1755,46 @@ pub async fn create_custom_instance_database( error::Error::internal_err(format!("Failed to create database '{}': {}", dbname, e)) })?; + // Nothing names a database that failed past this point, and its name blocks the retry: drop it + // rather than leave it behind. + if let Err(e) = finish_custom_instance_database(db, dbname, tag).await { + match drop_unused_instance_database(db, dbname).await { + Ok(Cleanup::InUse(users)) => tracing::warn!( + "Kept '{dbname}' after failing to set it up: workspaces {} use it", + users.join(", ") + ), + Ok(Cleanup::Waiter(pid)) => tracing::warn!( + "Kept '{dbname}' after failing to set it up: a request (pid {pid}) is still \ + waiting to name it. Drop it from instance settings once it is unused." + ), + Ok(Cleanup::Dropped) => {} + Err(drop_err) => { + tracing::error!("Could not drop '{dbname}' after failing to set it up: {drop_err}") + } + } + return Err(e); + } + + // A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT + // would otherwise let every role in regardless of what this instance defines. Best-effort: a + // failure here leaves the database usable as `admin`, and the next role change repairs it. + if let Err(e) = crate::datatable_roles::converge_connect_grants(db, dbname).await { + tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}"); + } + + tracing::info!("Created custom instance database '{}'", dbname); + Ok(()) +} + +/// Grant `custom_instance_user` its privileges on a database just created, and register it. +async fn finish_custom_instance_database(db: &DB, dbname: &str, tag: &str) -> error::Result<()> { // Grant permissions to custom_instance_user let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; let new_pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds }; let (client, connection) = new_pg_creds.connect(Some(db)).await?; let join_handle = tokio::spawn(async move { connection.await }); - if let Err(e) = client - .batch_execute(&format!( - "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; - GRANT USAGE ON SCHEMA public TO custom_instance_user; - GRANT CREATE ON SCHEMA public TO custom_instance_user; - GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;" - )) - .await - { + if let Err(e) = client.batch_execute(&instance_db_grants(dbname)).await { tracing::warn!( "Failed to grant permissions on '{}': {}. Continuing.", dbname, @@ -1473,8 +1822,6 @@ pub async fn create_custom_instance_database( ) .execute(db) .await?; - - tracing::info!("Created custom instance database '{}'", dbname); Ok(()) } @@ -1782,11 +2129,9 @@ pub async fn on_behalf_of_from_permissioned_as( let Some(permissioned_as) = permissioned_as else { return Ok(None); }; - // Uncached: the address is copied onto the job row, where it stays for the life of the run - // and decides the superadmin flag and the instance groups. Nothing evicts the cache across - // processes, so a cached read would keep minting jobs under an address the account no longer - // holds for up to a minute after it moves. - let email = users::get_email_from_permissioned_as_uncached(permissioned_as, w_id, db).await?; + // Cached on purpose, up to one notify poll stale: the accepted dispatch case + // `get_email_from_permissioned_as` documents. + let email = users::get_email_from_permissioned_as(permissioned_as, w_id, db).await?; Ok(Some(jobs::OnBehalfOf { email, permissioned_as: permissioned_as.to_string(), diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index 60d74f189d..9d85278404 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -329,6 +329,7 @@ pub fn try_expand_internal_db_query( "ALTER_TABLE" => expand_alter_table(json_str, db_type).map(ExpandedQuery::sql), "CREATE_SCHEMA" => expand_create_schema(json_str, db_type).map(ExpandedQuery::sql), "DROP_SCHEMA" => expand_drop_schema(json_str, db_type).map(ExpandedQuery::sql), + "RENAME_SCHEMA" => expand_rename_schema(json_str, db_type).map(ExpandedQuery::sql), // Metadata queries "LOAD_TABLE_METADATA" => expand_load_table_metadata(json_str, db_type), "FOREIGN_KEYS" => expand_foreign_keys(json_str, db_type).map(ExpandedQuery::sql), @@ -1716,6 +1717,13 @@ struct DropSchemaPayload { ducklake: Option, } +#[derive(Deserialize)] +struct RenameSchemaPayload { + schema: String, + new_schema: String, + ducklake: Option, +} + #[derive(Debug, Clone, Deserialize)] struct TableEditorColumn { name: String, @@ -2004,6 +2012,23 @@ fn expand_drop_schema(json_str: &str, db_type: DbType) -> Result Ok(maybe_wrap_ducklake(query, p.ducklake.as_deref())) } +fn expand_rename_schema(json_str: &str, db_type: DbType) -> Result { + let p: RenameSchemaPayload = serde_json::from_str(json_str) + .map_err(|e| format!("Invalid RENAME_SCHEMA payload: {}", e))?; + if !matches!(db_type, DbType::Postgresql | DbType::Snowflake) || p.ducklake.is_some() { + return Err(format!( + "Renaming a schema is not supported on {:?}", + db_type + )); + } + let query = format!( + "ALTER SCHEMA {} RENAME TO {};", + qi(&p.schema, db_type), + qi(&p.new_schema, db_type) + ); + Ok(query) +} + fn expand_create_table(json_str: &str, db_type: DbType) -> Result { let p: CreateTablePayload = serde_json::from_str(json_str) .map_err(|e| format!("Invalid CREATE_TABLE payload: {}", e))?; @@ -2598,7 +2623,9 @@ WHERE table_catalog = current_database()", ) } else { ( - "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'".to_string(), + // pg_catalog is readable by everyone: without the privilege check this lists + // tables of schemas the connection's role cannot even enter. + "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'\n AND has_schema_privilege(ns.oid, 'USAGE')".to_string(), ",\n ns.nspname AS schema_name,\n c.relname AS table_name".to_string(), "\nJOIN pg_catalog.pg_class c ON a.attrelid = c.oid\nJOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid".to_string(), "ns.nspname, c.relname, a.attnum".to_string(), @@ -4101,6 +4128,13 @@ mod tests { assert_eq!(sql, "DROP SCHEMA \"old_schema\" CASCADE;"); } + #[test] + fn test_expand_rename_schema() { + let marker = r#"-- WM_INTERNAL_DB_RENAME_SCHEMA {"schema":"old","new_schema":"new"}"#; + let sql = expand_code(marker, &ScriptLang::Postgresql); + assert_eq!(sql, "ALTER SCHEMA \"old\" RENAME TO \"new\";"); + } + #[test] fn test_expand_create_schema_with_ducklake() { let marker = r#"-- WM_INTERNAL_DB_CREATE_SCHEMA {"schema":"s","ducklake":"lake"}"#; @@ -4468,6 +4502,7 @@ mod tests { assert!(sql.contains("schema_name")); assert!(sql.contains("table_name")); assert!(sql.contains("c.relkind = 'r'")); + assert!(sql.contains("has_schema_privilege(ns.oid, 'USAGE')")); } #[test] diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs index 2656ce5af5..2008e1ce93 100644 --- a/backend/windmill-common/src/queue.rs +++ b/backend/windmill-common/src/queue.rs @@ -15,6 +15,57 @@ pub async fn get_queue_counts(db: &Pool) -> HashMap { .unwrap_or_else(|| HashMap::new()) } +/// Backlog of a single tag: jobs waiting more than 3 seconds past their `scheduled_for`. +pub struct QueueStat { + pub count: u32, + /// How long the job that would be picked up next has already been waiting, in seconds. + pub delay: f64, + /// When that job started waiting (its `scheduled_for`), in epoch seconds. + pub head_since: f64, +} + +/// Same backlog as [`get_queue_counts`], plus the delay of the job at the head of each +/// tag's queue. The head is picked with the same ordering the worker pull uses, so the +/// delay reported is the one a worker is about to observe. +/// +/// Reads the queue of every workspace: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_counts` does. Unlike [`get_queue_counts`], a failed +/// read is an error rather than an empty map, which would read as every backlog draining. +pub async fn get_queue_stats( + db: &Pool, +) -> crate::error::Result> { + // Grouping by (tag, priority) first finds every head in the same single pass as the + // count. A per-tag `ORDER BY ... LIMIT 1` walks `queue_sort_v2`, whose `tag` column comes + // last, through every other tag's backlog queued ahead of it. + let rows = sqlx::query!( + "SELECT tag AS \"tag!\", count AS \"count!\", + EXTRACT(EPOCH FROM now() - head)::double precision AS \"delay!\", + EXTRACT(EPOCH FROM head)::double precision AS \"head_since!\" + FROM ( + SELECT tag, sum(n)::bigint AS count, + (array_agg(head ORDER BY priority DESC NULLS LAST))[1] AS head + FROM ( + SELECT tag, priority, count(*) AS n, min(scheduled_for) AS head + FROM v2_job_queue WHERE + scheduled_for <= now() - ('3 seconds')::interval AND running = false + GROUP BY tag, priority + ) g + GROUP BY tag + ) t", + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|x| { + ( + x.tag, + QueueStat { count: x.count as u32, delay: x.delay, head_since: x.head_since }, + ) + }) + .collect()) +} + pub async fn get_queue_running_counts(db: &Pool) -> HashMap { sqlx::query!( "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE diff --git a/backend/windmill-common/src/queue_metrics.rs b/backend/windmill-common/src/queue_metrics.rs new file mode 100644 index 0000000000..f5affd7346 --- /dev/null +++ b/backend/windmill-common/src/queue_metrics.rs @@ -0,0 +1,482 @@ +//! The queue metrics the monitor samples into `metrics` (`queue_count_{tag}` and +//! `queue_delay_{tag}`), and how a stored series is drawn back. +//! +//! A stored value is a number, held until the next sample, or, for a delay, `{"since": }`: the job at the head of the queue has been waiting since then and was still there +//! when sampled, so the delay climbs one second per second until the next sample. Besides +//! [`QueueSample`], the SQL in [`read_queue_metrics_series`] and in `GET /workers/queue_metrics` +//! decodes both shapes. + +use std::collections::BTreeMap; + +use serde::Serialize; +use sqlx::{Pool, Postgres}; + +pub const QUEUE_COUNT_PREFIX: &str = "queue_count_"; +pub const QUEUE_DELAY_PREFIX: &str = "queue_delay_"; + +/// A backlogged tag whose value has not moved is re-sampled only this often. A longer heartbeat +/// writes fewer rows, but keeps a tag whose drain was never recorded (no server was up when it +/// drained) drawn as backlogged for longer. +pub const QUEUE_METRIC_HEARTBEAT_SECS: f64 = 5.0 * 60.0; + +/// A series silent for longer than this has drained: the sampler stops looking for it, so no +/// closing zero will come, and it is drawn as zero from there. Heartbeats land up to a monitor +/// tick and a sampling slot late, so this must stay well above their real spacing. +pub const QUEUE_METRIC_STALE_SECS: f64 = 3.0 * QUEUE_METRIC_HEARTBEAT_SECS; + +/// Heads that started waiting within this of each other are one wait: jobs queued together +/// leave the head one after another without the delay dropping. +pub const QUEUE_DELAY_SAME_HEAD_SECS: f64 = 1.0; + +/// Slots a series is split into, whatever the window. A slot draws at most four vertices, and a +/// climb one more at each slot boundary it crosses, so a line stays under about 600 points +/// however many rows the window holds. +const QUEUE_METRICS_SERIES_SLOTS: f64 = 120.0; + +/// A stored sample, as it is drawn from the moment it was written until the next one. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum QueueSample { + /// A count, or a delay while the head keeps changing, which hovers. + Held(f64), + /// A delay while the job that started waiting at `since` (epoch seconds) stays at the head. + Climbing { since: f64 }, +} + +impl QueueSample { + pub fn parse(value: &serde_json::Value) -> Option { + match value.get("since") { + Some(since) => since.as_f64().map(|since| Self::Climbing { since }), + None => value.as_f64().map(Self::Held), + } + } + + pub fn to_json(self) -> serde_json::Value { + match self { + Self::Held(value) => serde_json::json!(value), + Self::Climbing { since } => serde_json::json!({ "since": since }), + } + } + + /// Its value at `t`, in epoch seconds. + pub fn value_at(self, t: f64) -> f64 { + match self { + Self::Held(value) => value, + Self::Climbing { since } => t - since, + } + } + + /// When the job at the head of a delay sample written at `at` started waiting. + pub fn head_since(self, at: f64) -> f64 { + match self { + Self::Held(delay) => at - delay, + Self::Climbing { since } => since, + } + } +} + +#[derive(Serialize)] +pub struct QueueMetricsSeries { + /// The window drawn, in epoch milliseconds. + pub from: i64, + pub to: i64, + pub tags: Vec, +} + +#[derive(Serialize)] +pub struct QueueTagSeries { + pub tag: String, + /// Vertices `[epoch ms, value]` of a line joined by straight segments. + pub count: Vec<(i64, f64)>, + pub delay: Vec<(i64, f64)>, +} + +/// The queue metrics of the last `window_secs`, each series aggregated per slot by the database +/// and drawn by [`render_series`], so the size is bounded by the number of tags rather than by +/// how many rows they wrote. +/// +/// Reads the metrics of every workspace's tags: a caller exposing the result MUST restrict it to +/// devops users, as `GET /workers/queue_metrics_series` does. +pub async fn read_queue_metrics_series( + db: &Pool, + window_secs: f64, +) -> crate::error::Result { + let to = sqlx::query_scalar!("SELECT EXTRACT(EPOCH FROM now())::double precision AS \"now!\"") + .fetch_one(db) + .await?; + let from = to - window_secs; + let slot_secs = window_secs / QUEUE_METRICS_SERIES_SLOTS; + + // Slot -1 holds the samples written before the window, of which only the last is used: it + // sets the value in force at the left edge. A series silent for longer than the stale window + // reads as zero, so nothing older can matter. Arrays compare element by element, so + // `max(ARRAY[t, v])` is the slot's latest sample, found without sorting every row. `v` is a + // sample's value when it was written: for a climbing delay, how long its head had waited. + // + // A climb keeps rising until the next sample, so when that sample lands in the same slot + // (the tag drained, or its head moved), the climb's top is higher than any `v`. Looking the + // next sample up for the slot's last climb, rather than ordering every row, keeps the pass a + // plain aggregate; an earlier climb in the same slot still shows up to its last heartbeat. + // `t` round-trips through `to_timestamp` to within a microsecond either way, so both bounds + // carry a millisecond of slack, far less than two distinct samples of a series are apart: + // without it the climbing sample can match itself, or the one at `last` fall outside. + let rows = sqlx::query!( + "WITH slots AS ( + SELECT id, slot, min(t) AS first, max(t) AS last, max(v) AS peak, + (min(ARRAY[t, v]))[2] AS first_value, (max(ARRAY[t, v]))[2] AS last_value, + (max(ARRAY[t, climbing]))[2] = 1 AS last_climbing, + COALESCE(bool_and(climbing = 1) AND max(since) - min(since) < $4, false) AS ramp, + max(ARRAY[t, since]) FILTER (WHERE climbing = 1) AS last_climb + FROM ( + SELECT id, t, + CASE jsonb_typeof(value) + WHEN 'number' THEN value::double precision + WHEN 'object' THEN t - (value->>'since')::double precision + END AS v, + (value->>'since')::double precision AS since, + (jsonb_typeof(value) = 'object')::int::double precision AS climbing, + greatest(floor((t - $1::double precision) / $2::double precision), -1)::int + AS slot + FROM ( + SELECT id, value, EXTRACT(EPOCH FROM created_at)::double precision AS t + FROM metrics + WHERE id LIKE 'queue_%' + AND created_at > to_timestamp($1::double precision - $3::double precision) + ) m + ) s + WHERE v IS NOT NULL + GROUP BY id, slot + ) + SELECT id AS \"id!\", slot AS \"slot!\", first AS \"first!\", last AS \"last!\", + greatest(peak, CASE WHEN last_climb[1] < last THEN ( + SELECT EXTRACT(EPOCH FROM min(n.created_at))::double precision + FROM metrics n + WHERE n.id = slots.id AND n.id LIKE 'queue_%' + AND n.created_at > to_timestamp(last_climb[1] + 0.001) + AND n.created_at <= to_timestamp(last + 0.001) + ) - last_climb[2] END) AS \"peak!\", + first_value AS \"first_value!\", last_value AS \"last_value!\", + last_climbing AS \"last_climbing!\", ramp AS \"ramp!\" + FROM slots + ORDER BY id, slot", + from, + slot_secs, + QUEUE_METRIC_STALE_SECS, + QUEUE_DELAY_SAME_HEAD_SECS, + ) + .fetch_all(db) + .await?; + + #[derive(Default)] + struct Stored { + carried: Option, + slots: Vec, + } + // [count, delay] per tag. + let mut stored: BTreeMap = BTreeMap::new(); + for row in rows { + let (series, tag) = if let Some(tag) = row.id.strip_prefix(QUEUE_COUNT_PREFIX) { + (0, tag) + } else if let Some(tag) = row.id.strip_prefix(QUEUE_DELAY_PREFIX) { + (1, tag) + } else { + continue; + }; + let series = &mut stored.entry(tag.to_string()).or_default()[series]; + let slot = MetricSlot { + first: row.first, + last: row.last, + peak: row.peak, + first_value: row.first_value, + last_value: row.last_value, + last_climbing: row.last_climbing, + ramp: row.ramp, + }; + if row.slot < 0 { + series.carried = Some(slot); + } else { + series.slots.push(slot); + } + } + + let tags = stored + .into_iter() + .map(|(tag, [count, delay])| { + let draw = + |s: &Stored| render_series(s.carried.as_ref(), &s.slots, from, to, slot_secs); + QueueTagSeries { count: draw(&count), delay: draw(&delay), tag } + }) + // A tag that drained before the window has nothing to draw in it. + .filter(|s| s.count.iter().chain(&s.delay).any(|(_, v)| *v != 0.0)) + .collect(); + + Ok(QueueMetricsSeries { + from: (from * 1000.0).round() as i64, + to: (to * 1000.0).round() as i64, + tags, + }) +} + +/// The stored samples of one series that fall in one time slot. +#[derive(Debug, Clone, Copy)] +pub struct MetricSlot { + /// When the first and the last sample of the slot were written, in epoch seconds. + pub first: f64, + pub last: f64, + /// The highest value the series drew over the slot, a climb that ends inside it included. + pub peak: f64, + pub first_value: f64, + /// The value of the last sample, which holds (or climbs, for a climbing delay) until the next. + pub last_value: f64, + pub last_climbing: bool, + /// Every sample of the slot climbs from the same head, so the slot is one exact ramp. + pub ramp: bool, +} + +/// Draw a stored series over `[from, to]` (epoch seconds), split into slots of `slot_secs`, as +/// the vertices of a line joined by straight segments, each `(epoch ms, value)`. +/// +/// A sample holds its value, or a climbing delay keeps climbing, until the next sample or until +/// the series has been silent for [`QUEUE_METRIC_STALE_SECS`]. `carried` is the slot before +/// `from`, whose last sample sets the left edge. A slot draws its peak across the span of its +/// samples, so a spike shorter than a slot still shows at full height, unless it is a single +/// climb, drawn exactly. A climb gets a vertex at every slot boundary it crosses: the delay axis +/// is logarithmic, so one straight segment across many slots would misplace it. +pub fn render_series( + carried: Option<&MetricSlot>, + slots: &[MetricSlot], + from: f64, + to: f64, + slot_secs: f64, +) -> Vec<(i64, f64)> { + let mut line = Line { points: vec![], from, slot_secs }; + let mut held = carried + .map(Held::after) + .filter(|h| from - h.at <= QUEUE_METRIC_STALE_SECS); + if let Some(h) = held { + line.push(from, h.value_at(from)); + } + for slot in slots { + let entering = line.advance(&mut held, slot.first); + line.push(slot.first, entering); + if slot.ramp { + line.push(slot.first, slot.first_value); + } else { + line.push(slot.first, slot.peak); + line.push(slot.last, slot.peak); + } + line.push(slot.last, slot.last_value); + held = Some(Held::after(slot)); + } + if !line.points.is_empty() { + let value = line.advance(&mut held, to); + line.push(to, value); + } + line.points +} + +/// The last sample drawn: when it was written, its value then, and whether it climbs from there. +#[derive(Clone, Copy)] +struct Held { + at: f64, + value: f64, + climbing: bool, +} + +impl Held { + fn after(slot: &MetricSlot) -> Self { + Self { at: slot.last, value: slot.last_value, climbing: slot.last_climbing } + } + + fn value_at(self, t: f64) -> f64 { + if self.climbing { + self.value + (t - self.at) + } else { + self.value + } + } +} + +struct Line { + points: Vec<(i64, f64)>, + from: f64, + slot_secs: f64, +} + +impl Line { + /// The value `held` has at `t`, drawing the climb that leads there and, when the series went + /// silent for too long first, its drop to zero, after which it is forgotten. + fn advance(&mut self, held: &mut Option, t: f64) -> f64 { + let Some(h) = *held else { + return 0.0; + }; + let stale_at = h.at + QUEUE_METRIC_STALE_SECS; + if h.climbing { + let end = t.min(stale_at); + let start = h.at.max(self.from); + let mut boundary = self.from + + ((start - self.from) / self.slot_secs).floor() * self.slot_secs + + self.slot_secs; + while boundary < end { + self.push(boundary, h.value_at(boundary)); + boundary += self.slot_secs; + } + } + if t <= stale_at { + return h.value_at(t); + } + self.push(stale_at, h.value_at(stale_at)); + self.push(stale_at, 0.0); + *held = None; + 0.0 + } + + fn push(&mut self, t: f64, value: f64) { + let point = ((t * 1000.0).round() as i64, value); + match self.points.as_mut_slice() { + [.., last] if *last == point => {} + // A horizontal run only needs its two ends. + [.., a, b] if a.1 == value && b.1 == value => b.0 = point.0, + _ => self.points.push(point), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const FROM: f64 = 1_000_000.0; + const TO: f64 = FROM + 3600.0; + const SLOT: f64 = 30.0; + + fn held(first: f64, last: f64, peak: f64, last_value: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak, + first_value: peak, + last_value, + last_climbing: false, + ramp: false, + } + } + + /// A slot whose samples all climb from a head that started waiting 30s before `FROM`. + fn climbing(first: f64, last: f64) -> MetricSlot { + MetricSlot { + first: FROM + first, + last: FROM + last, + peak: last + 30.0, + first_value: first + 30.0, + last_value: last + 30.0, + last_climbing: true, + ramp: true, + } + } + + fn at(secs: f64, value: f64) -> (i64, f64) { + (((FROM + secs) * 1000.0) as i64, value) + } + + #[test] + fn a_value_holds_until_the_next_sample_and_a_drain_drops_where_it_was_written() { + let line = render_series( + None, + &[ + held(60.0, 60.0, 3.0, 3.0), + held(600.0, 600.0, 2.0, 2.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + SLOT, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(600.0, 3.0), + at(600.0, 2.0), + at(900.0, 2.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } + + #[test] + fn a_series_silent_past_the_stale_window_drops_to_zero() { + let line = render_series(None, &[held(60.0, 60.0, 3.0, 3.0)], FROM, TO, SLOT); + let dropped = 60.0 + QUEUE_METRIC_STALE_SECS; + assert_eq!( + line, + vec![ + at(60.0, 0.0), + at(60.0, 3.0), + at(dropped, 3.0), + at(dropped, 0.0), + at(3600.0, 0.0) + ] + ); + } + + #[test] + fn a_slot_draws_its_peak_then_continues_from_its_last_sample() { + // Samples at 60 (5), 70 (9), 80 (4) collapsed into one slot. + let line = render_series( + Some(&held(-30.0, -30.0, 2.0, 2.0)), + &[held(60.0, 80.0, 9.0, 4.0)], + FROM, + FROM + 120.0, + SLOT, + ); + assert_eq!( + line, + vec![ + at(0.0, 2.0), + at(60.0, 2.0), + at(60.0, 9.0), + at(80.0, 9.0), + at(80.0, 4.0), + at(120.0, 4.0) + ] + ); + } + + #[test] + fn a_climbing_delay_is_drawn_exactly_up_to_its_drain() { + // 300s slots: one holds two climbing samples, and heartbeats follow until the drain. + let line = render_series( + None, + &[ + climbing(60.0, 120.0), + climbing(360.0, 360.0), + climbing(660.0, 660.0), + held(900.0, 900.0, 0.0, 0.0), + ], + FROM, + TO, + 300.0, + ); + assert_eq!( + line, + vec![ + at(60.0, 0.0), + // The slot is one climb, not its peak held across it. + at(60.0, 90.0), + at(120.0, 150.0), + // A vertex at each slot boundary the climb crosses. + at(300.0, 330.0), + at(360.0, 390.0), + at(600.0, 630.0), + at(660.0, 690.0), + // Still climbing right up to the closing zero. + at(900.0, 930.0), + at(900.0, 0.0), + at(3600.0, 0.0), + ] + ); + } +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 31219621e9..628ae2a709 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -91,9 +91,6 @@ async fn prefetch_cached_script_inner( derive_email: bool, ) -> crate::error::Result> { let derived_email = match script.on_behalf_of.as_deref().filter(|_| derive_email) { - // Uncached: the client preserves this pair and sends it back, where the write path - // validates it against an uncached lookup. A cached address would pair a live principal - // with an address the account no longer holds, and the redeploy would be rejected. Some(permissioned_as) => Some( crate::users::get_email_from_permissioned_as_uncached( permissioned_as, @@ -250,23 +247,37 @@ pub async fn get_full_hub_script_by_path( let version = path_iterator .next() .ok_or_else(|| Error::internal_err(format!("expected hub path to have version number")))?; + // A cache entry that cannot be read or parsed counts as a miss rather than an error: + // a truncated write leaves a file that exists but deserializes to nothing, and refetching + // it is always preferable to failing the job push it was read for. let cache_path = format!("{}/{version}", *HUB_CACHE_DIR); - let script; - if tokio::fs::metadata(&cache_path).await.is_err() { - script = get_full_hub_script_by_path_inner(path, http_client, db).await?; - if let Err(e) = crate::worker::write_file( - &HUB_CACHE_DIR, - &version, - &serde_json::to_string(&script).map_err(to_anyhow)?, - ) { - tracing::error!("failed to write hub script {path} to cache: {e}"); - } else { - tracing::info!("wrote hub script {path} to cache"); + let cached = match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => serde_json::from_str::(&content) + .inspect_err(|e| { + tracing::error!("hub script cache at {cache_path} is unparseable, refetching: {e}") + }) + .ok(), + Err(e) => { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::error!("hub script cache at {cache_path} is unreadable, refetching: {e}"); + } + None } - } else { - let cache_content = tokio::fs::read_to_string(cache_path).await?; - script = serde_json::from_str(&cache_content).unwrap(); + }; + if let Some(script) = cached { tracing::info!("read hub script {path} from cache"); + return Ok(script); + } + + let script = get_full_hub_script_by_path_inner(path, http_client, db).await?; + if let Err(e) = crate::worker::write_file( + &HUB_CACHE_DIR, + &version, + &serde_json::to_string(&script).map_err(to_anyhow)?, + ) { + tracing::error!("failed to write hub script {path} to cache: {e}"); + } else { + tracing::info!("wrote hub script {path} to cache"); } Ok(script) } @@ -358,31 +369,38 @@ pub async fn fetch_script_for_update<'a>( .map_err(crate::error::Error::from) } -pub struct ClonedScript { - pub old_script: NewScript, - pub new_hash: i64, -} -// TODO: What if dependency job fails, there is script with NULL in the lock -pub async fn clone_script<'c>( - path: &str, - w_id: &str, +/// Deploys the outcome of a relative-import relock as a new version of `head`, the path's live +/// version that the caller holds `FOR UPDATE`, and archives `head`. A `lock` of `None` records +/// a failed generation: the version carries `lock_error_logs` instead and runs keep resolving +/// to the last version that has a lock. A `modules` of `None` keeps the head's module locks. +/// +/// Writes whatever `head` names and checks nothing: callers are responsible for having +/// established access to its workspace and path, as a dependency job's push already has. +/// +/// `created_at` is stamped when the insert runs, not at transaction start. The row lock on +/// `head` is what orders one relock after another, and with `now()` a transaction that began +/// first but locked second commits a live child older than its archived parent, which every +/// "latest version" read then mis-orders. +pub async fn deploy_relocked_version( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + head: Script, deployment_message: Option, - db: &DB, -) -> crate::error::Result { - let mut tx = db.begin().await?; - let s = if let Some(s) = fetch_script_for_update(path, w_id, &mut *tx).await? { - s - } else { - return Err(crate::error::Error::NotFound(format!( - "Non-archived script with path '{}' not found", - path - ))); - }; + lock: Option<&str>, + modules: Option<&std::collections::HashMap>, + lock_error_logs: Option<&str>, +) -> crate::error::Result { + let s = head; + let w_id = s.workspace_id.as_str(); - let rs = runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut *tx) - .await?; + let rs = + runnable_settings::from_handle(s.runnable_settings.runnable_settings_handle, &mut **tx) + .await?; let (debouncing_settings, concurrency_settings) = - runnable_settings::prefetch_cached_tx(&rs, &mut tx).await?; + runnable_settings::prefetch_cached_tx(&rs, &mut *tx).await?; + + // What the row stores is what the hash covers: the new module locks when there are any. + let modules = modules.cloned().or(s.modules); + let modules_json = modules.as_ref().map(serde_json::to_value).transpose()?; let ns = NewScript { path: s.path.clone(), @@ -392,7 +410,7 @@ pub async fn clone_script<'c>( content: s.content, schema: s.schema, is_template: s.is_template, - lock: None, + lock: lock.map(str::to_string), language: s.language, kind: Some(s.kind), tag: s.tag, @@ -424,7 +442,7 @@ pub async fn clone_script<'c>( on_behalf_of: s.on_behalf_of, preserve_on_behalf_of: None, assets: s.assets, - modules: s.modules, + modules, auto_parent: None, labels: s.labels, skip_draft_deletion: None, @@ -433,7 +451,7 @@ pub async fn clone_script<'c>( let new_hash = hash_script(&ns); tracing::debug!( - "cloning script at path {} from '{}' to '{}'", + "deploying relocked version of script at path {} from '{}' to '{}'", s.path, *s.hash, new_hash @@ -446,17 +464,19 @@ pub async fn clone_script<'c>( envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels) + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, \ + lock_error_logs, created_at) SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ - content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ + content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, \ envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ - codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels + codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, \ + $6::text, clock_timestamp() FROM script WHERE hash = $2 AND workspace_id = $3; - ", new_hash, s.hash.0, w_id).execute(&mut *tx).await?; + ", new_hash, s.hash.0, w_id, lock, modules_json, lock_error_logs).execute(&mut **tx).await?; // Archive base. sqlx::query!( @@ -464,9 +484,8 @@ pub async fn clone_script<'c>( *s.hash, w_id ) - .execute(&mut *tx) + .execute(&mut **tx) .await?; - tx.commit().await?; - Ok(ClonedScript { old_script: ns, new_hash }) + Ok(new_hash) } diff --git a/backend/windmill-common/src/sensitive_log_masks.rs b/backend/windmill-common/src/sensitive_log_masks.rs index b6f6262b77..c999297ec0 100644 --- a/backend/windmill-common/src/sensitive_log_masks.rs +++ b/backend/windmill-common/src/sensitive_log_masks.rs @@ -10,7 +10,7 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use uuid::Uuid; /// Minimum length for a secret to be registered for masking. @@ -20,9 +20,27 @@ const MIN_SECRET_LENGTH: usize = 8; const MASKED_NOTICE: &str = "[windmill] secret value was masked for security reasons, use string transformations to display full value"; +/// The secrets registered for one job, plus the automaton compiled from them. +#[derive(Default)] +struct JobMasks { + secrets: HashSet, + /// Built on the first `snapshot` after a change and shared by every later + /// snapshot. Every job registers at least its own token, so without this + /// cache each log batch of each job would rebuild the automaton. + compiled: Option>, +} + +/// Aho-Corasick automaton for O(m) multi-pattern matching in a single pass, +/// regardless of the number of secrets registered, with the replacement +/// strings indexed to match the automaton's pattern order. +struct CompiledMasks { + ac: aho_corasick::AhoCorasick, + replacements: Vec, +} + lazy_static::lazy_static! { - /// Map of job_id -> set of secret values that should be masked in that job's logs. - static ref SENSITIVE_MASKS: RwLock>> = + /// Map of job_id -> secret values that should be masked in that job's logs. + static ref SENSITIVE_MASKS: RwLock> = RwLock::new(HashMap::new()); /// Set of currently running job IDs on this worker process. @@ -32,13 +50,8 @@ lazy_static::lazy_static! { } /// A lock-free snapshot of secrets for a job, taken once per log batch. -/// Uses Aho-Corasick for O(m) multi-pattern matching in a single pass, -/// regardless of the number of secrets registered. pub struct MaskSnapshot { - /// Aho-Corasick automaton for fast matching. - ac: aho_corasick::AhoCorasick, - /// Replacement strings, indexed to match the automaton's pattern order. - replacements: Vec, + compiled: Arc, /// Whether the security notice has already been appended for this snapshot. /// Tracked locally to avoid a global write lock on every masked line. notice_shown: std::cell::Cell, @@ -53,34 +66,104 @@ impl MaskSnapshot { } // Single-pass check + replace using the pre-built automaton - if !self.ac.is_match(text) { + if !self.compiled.ac.is_match(text) { return Cow::Borrowed(text); } - let mut result = self.ac.replace_all(text, &self.replacements); + let mut result = self + .compiled + .ac + .replace_all(text, &self.compiled.replacements); - // Append the notice only once per snapshot (i.e. per batch) + // Append the notice only once per snapshot (i.e. per batch), as its own line. + // Callers pass either a bare line (`handle_child`) or a chunk that already ends + // in a newline (nativets), and the sinks concatenate what they get verbatim: + // assuming either shape welds the notice onto a neighbouring line. if !self.notice_shown.get() { self.notice_shown.set(true); - result.push('\n'); - result.push_str(MASKED_NOTICE); + if result.ends_with('\n') { + result.push_str(MASKED_NOTICE); + result.push('\n'); + } else { + result.push('\n'); + result.push_str(MASKED_NOTICE); + } } Cow::Owned(result) } } +/// A masker for sinks that mask line by line rather than in batches, like nativets +/// masking each `console.log` chunk as V8 produces it. `snapshot` per line would +/// re-arm the security notice on every one; this keeps it to once per distinct set +/// of secrets while still picking up secrets registered mid-run. +/// +/// Masks by job id alone — the caller is the one that knows the text it passes +/// belongs to that job. +pub struct JobMasker { + job_id: Uuid, + snapshot: Option, +} + +impl JobMasker { + pub fn new(job_id: Uuid) -> Self { + JobMasker { job_id, snapshot: snapshot(&job_id) } + } + + /// Mask every secret registered for the job. Returns `Cow::Borrowed` when no match. + /// Falls back to the masks it last saw once the job is unregistered, so a sink + /// still draining past the end of a run does not start emitting secrets. + pub fn mask<'a>(&mut self, text: &'a str) -> Cow<'a, str> { + if let Some(fresh) = snapshot(&self.job_id) { + // Replacing an equivalent snapshot would re-arm the notice, so only take + // one built from a secret set we have not seen. + let unchanged = self + .snapshot + .as_ref() + .is_some_and(|cur| Arc::ptr_eq(&cur.compiled, &fresh.compiled)); + if !unchanged { + self.snapshot = Some(fresh); + } + } + match self.snapshot.as_ref() { + Some(snapshot) => snapshot.mask(text), + None => Cow::Borrowed(text), + } + } +} + /// Take a snapshot of the current secrets for a job. Returns `None` if no secrets /// are registered (the caller can then skip masking entirely for the whole batch). /// /// Call this once per log batch in `write_lines`, not per line. pub fn snapshot(job_id: &Uuid) -> Option { - let masks = SENSITIVE_MASKS.read().unwrap_or_else(|e| e.into_inner()); - let secrets = masks.get(job_id)?; - if secrets.is_empty() { - return None; + { + let masks = SENSITIVE_MASKS.read().unwrap_or_else(|e| e.into_inner()); + let job = masks.get(job_id)?; + if job.secrets.is_empty() { + return None; + } + if let Some(compiled) = job.compiled.as_ref() { + return Some(MaskSnapshot { + compiled: compiled.clone(), + notice_shown: std::cell::Cell::new(false), + }); + } } + let mut masks = SENSITIVE_MASKS.write().unwrap_or_else(|e| e.into_inner()); + let job = masks.get_mut(job_id)?; + if job.secrets.is_empty() { + return None; + } + let compiled = job + .compiled + .get_or_insert_with(|| Arc::new(compile(&job.secrets))); + Some(MaskSnapshot { compiled: compiled.clone(), notice_shown: std::cell::Cell::new(false) }) +} + +fn compile(secrets: &HashSet) -> CompiledMasks { // Sort longest-first so longer secrets are matched before shorter substrings let mut sorted: Vec<&String> = secrets.iter().collect(); sorted.sort_by(|a, b| b.len().cmp(&a.len())); @@ -106,7 +189,7 @@ pub fn snapshot(job_id: &Uuid) -> Option { .build(sorted.iter().map(|s| s.as_str())) .expect("failed to build aho-corasick automaton"); - Some(MaskSnapshot { ac, replacements, notice_shown: std::cell::Cell::new(false) }) + CompiledMasks { ac, replacements } } /// Register a job as currently running. Call this before `handle_queued_job`. @@ -148,20 +231,110 @@ pub fn register_secret_for_all_running_jobs(secret: &str) { let mut masks = SENSITIVE_MASKS.write().unwrap_or_else(|e| e.into_inner()); for job_id in job_ids { - if let Some(set) = masks.get_mut(&job_id) { - set.insert(secret.to_string()); + if let Some(job) = masks.get_mut(&job_id) { + if job.secrets.insert(secret.to_string()) { + job.compiled = None; + } } } } /// Register a secret value for a specific job. -/// Used for `$encrypted:` args where we know the job ID. +/// Used for the job's own token and for `$encrypted:` args, where we know the job ID. pub fn register_secret_for_job(job_id: Uuid, secret: &str) { if secret.len() < MIN_SECRET_LENGTH { return; } let mut masks = SENSITIVE_MASKS.write().unwrap_or_else(|e| e.into_inner()); - if let Some(set) = masks.get_mut(&job_id) { - set.insert(secret.to_string()); + if let Some(job) = masks.get_mut(&job_id) { + if job.secrets.insert(secret.to_string()) { + job.compiled = None; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The compiled automaton is cached per job, so a secret registered after the + /// first snapshot only gets masked if the cache is invalidated. + #[test] + fn snapshot_rebuilds_after_a_new_secret_is_registered() { + let job_id = Uuid::new_v4(); + register_running_job(job_id); + register_secret_for_job(job_id, "firstsecretvalue"); + let _ = snapshot(&job_id) + .expect("secret registered") + .mask("firstsecretvalue"); + + register_secret_for_job(job_id, "secondsecretvalue"); + + let snap = snapshot(&job_id).expect("secrets registered"); + let masked = snap.mask("firstsecretvalue then secondsecretvalue"); + assert!(!masked.contains("firstsecretvalue"), "{masked}"); + assert!(!masked.contains("secondsecretvalue"), "{masked}"); + unregister_running_job(job_id); + } + + /// A line-by-line sink must not repeat the notice on every line, and must still + /// pick up a secret registered after the masker was built. + #[test] + fn job_masker_notices_once_per_secret_set() { + let job_id = Uuid::new_v4(); + register_running_job(job_id); + register_secret_for_job(job_id, "firstsecretvalue"); + let mut masker = JobMasker::new(job_id); + + let first = masker.mask("saw firstsecretvalue").into_owned(); + assert!(!first.contains("firstsecretvalue"), "{first}"); + assert!(first.contains(MASKED_NOTICE), "{first}"); + + let second = masker.mask("saw firstsecretvalue again").into_owned(); + assert!(!second.contains("firstsecretvalue"), "{second}"); + assert!(!second.contains(MASKED_NOTICE), "{second}"); + + register_secret_for_job(job_id, "secondsecretvalue"); + let third = masker.mask("saw secondsecretvalue").into_owned(); + assert!(!third.contains("secondsecretvalue"), "{third}"); + unregister_running_job(job_id); + } + + /// Unregistration must not turn masking off under a sink that is still emitting: + /// the masker keeps working off the masks it last saw rather than going quiet. + #[test] + fn job_masker_masks_after_the_job_is_unregistered() { + let job_id = Uuid::new_v4(); + register_running_job(job_id); + register_secret_for_job(job_id, "supersecretvalue"); + let mut masker = JobMasker::new(job_id); + + unregister_running_job(job_id); + + let masked = masker.mask("logged supersecretvalue here"); + assert!(!masked.contains("supersecretvalue"), "{masked}"); + } + + /// The notice has to end up on a line of its own for both shapes callers pass: + /// a bare line (`handle_child`) and a newline-terminated chunk (nativets). The + /// sinks concatenate what they are given verbatim, so getting this wrong welds + /// the notice onto whichever line follows it. + #[test] + fn notice_lands_on_its_own_line_for_both_caller_shapes() { + let job_id = Uuid::new_v4(); + register_running_job(job_id); + register_secret_for_job(job_id, "supersecretvalue"); + + let line = snapshot(&job_id) + .expect("secret registered") + .mask("tok supersecretvalue"); + assert_eq!(line, format!("tok s*****e\n{MASKED_NOTICE}")); + + let chunk = snapshot(&job_id) + .expect("secret registered") + .mask("tok supersecretvalue\n"); + assert_eq!(chunk, format!("tok s*****e\n{MASKED_NOTICE}\n")); + + unregister_running_job(job_id); } } diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 4c50f597ff..31dee749e6 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -6,6 +6,12 @@ pub const ALLOW_PRIVATE_MCP_SERVER_URLS_ENV: &str = "ALLOW_PRIVATE_MCP_SERVER_UR pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METADATA_URLS"; +pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; + +/// Lets every git call reach hosts on a private network, whoever it is made for. +/// Without it, [`private_git_host_allowed`] decides. +pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; + /// Why a URL failed SSRF validation. /// /// The distinction matters for callers that gate private endpoints behind a @@ -18,6 +24,9 @@ pub enum SsrfValidationError { InvalidUrl(String), /// Scheme is not `http`/`https`. DisallowedScheme(String), + /// The URL uses `http` where `https` is required (guest JWKS). The private-host opt-in + /// also permits `http`, so, unlike the other scheme errors, this one the flag can fix. + HttpsRequired, /// No host in the URL. MissingHost, /// DNS resolution failed for the host. @@ -37,6 +46,9 @@ impl std::fmt::Display for SsrfValidationError { f, "URL scheme '{s}' is not allowed, only http and https are permitted" ), + SsrfValidationError::HttpsRequired => { + write!(f, "URL must use https") + } SsrfValidationError::MissingHost => write!(f, "URL must have a host"), SsrfValidationError::ResolutionFailed { host, source } => { write!(f, "Failed to resolve host '{host}': {source}") @@ -195,6 +207,54 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } +fn allow_local_git_remotes() -> bool { + std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + +/// Who a git call is made for, which decides whether it may reach a host on a +/// private network. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GitRemoteCaller { + /// A workspace admin's request, or Windmill's own work (polling, webhook and + /// token upkeep, the merge request after a deploy), whose errors only admins read. + AdminOrSystem, + /// A request from anyone who is not a workspace admin. + NonAdmin, +} + +/// Whether a git call made for `caller` may reach a host on a private network. +/// +/// The refusal is for non-admins, who may not be able to run code (operators) +/// and would read git's error output back as a probe of the server's network. +/// An admin can run code, which reaches those hosts from a worker already. On a +/// cloud instance, where a workspace admin is anyone who signed up, every caller +/// is refused. +pub fn private_git_host_allowed(caller: GitRemoteCaller) -> bool { + git_host_policy_allows( + caller, + allow_local_git_remotes(), + *crate::worker::CLOUD_HOSTED, + ) +} + +fn git_host_policy_allows(caller: GitRemoteCaller, opted_in: bool, cloud_hosted: bool) -> bool { + opted_in || (caller == GitRemoteCaller::AdminOrSystem && !cloud_hosted) +} + +/// Appended to a refusal of a private git host, naming what would let `caller` +/// through. `None` where nothing an instance administrator sets would help. +pub fn private_git_host_hint(caller: GitRemoteCaller) -> Option { + (caller == GitRemoteCaller::NonAdmin && !*crate::worker::CLOUD_HOSTED).then(|| { + format!( + "Only workspace admins can reach a git server on a private network. To allow \ + every user, set the {ALLOW_LOCAL_GIT_REMOTES_ENV}=true environment variable on \ + the Windmill servers" + ) + }) +} + pub async fn validate_saml_metadata_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -213,6 +273,36 @@ pub async fn validate_saml_metadata_url(url: &str) -> Result Result { + let parsed = + url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; + + let allow_private = std::env::var(ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1"); + + match parsed.scheme() { + "https" => {} + // Plaintext HTTP only under the explicit operator opt-in that also allows private + // hosts (dev/loopback): the JWKS supplies the keys that authenticate guest JWTs, so an + // on-path attacker who could replace an http response could forge accepted tokens. + "http" if allow_private => {} + "http" => return Err(SsrfValidationError::HttpsRequired), + scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), + } + + let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + + if allow_private { + return Ok(ValidatedTarget::unpinned(host)); + } + + validate_url_for_ssrf(url).await +} + pub async fn validate_mcp_server_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -594,6 +684,15 @@ mod tests { )); } + #[test] + fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { + use GitRemoteCaller::{AdminOrSystem, NonAdmin}; + assert!(git_host_policy_allows(AdminOrSystem, false, false)); + assert!(!git_host_policy_allows(NonAdmin, false, false)); + assert!(!git_host_policy_allows(AdminOrSystem, false, true)); + assert!(git_host_policy_allows(NonAdmin, true, true)); + } + #[tokio::test] async fn saml_ssrf_error_message_includes_env_hint_only_for_private_urls() { let private_error = validate_url_for_ssrf("http://127.0.0.1/metadata") diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5091c7e16d..b456e370fc 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -178,56 +178,40 @@ pub fn initialize_tracing( .with(logs_bridge.with_filter(otel_logs_filter)) .with(opentelemetry_filtered); - match *JSON_FMT { - true => { - // Stdout layer with its own filter - let stdout_layer = json_layer() - .with_writer(std::io::stdout) - .flatten_event(true) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // The service log files are written to be indexed, not tailed, so they always carry the + // JSON format: it is what preserves level, target and the current span as fields rather + // than as text the index would have to recover by regex. JSON_FMT governs stdout only. + let file_layer = json_layer() + .with_writer(log_file_writer) + .flatten_event(true) + .with_filter(file_env_filter) + .with_filter(create_targets_filter(default_env_filter)); - // File layer with its own filter - let file_layer = json_layer() - .with_writer(log_file_writer) - .flatten_event(true) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // Boxed so both arms have one type: the file layer is a single value and could not + // otherwise be typed against two different subscriber stacks. + let stdout_layer = match *JSON_FMT { + true => json_layer() + .with_writer(std::io::stdout) + .flatten_event(true) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + false => compact_layer() + .with_writer(std::io::stdout) + .with_ansi(style.to_lowercase() != "never") + .with_file(true) + .with_line_number(true) + .with_target(false) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + }; - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - false => { - // Stdout layer with its own filter - let stdout_layer = compact_layer() - .with_writer(std::io::stdout) - .with_ansi(style.to_lowercase() != "never") - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - // File layer with its own filter - let file_layer = compact_layer() - .with_writer(log_file_writer) - .with_ansi(false) // No ANSI codes in log files - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - } + base_layer + .with(stdout_layer) + .with(file_layer) + .with(CountingLayer::new()) + .init(); (_guard, meter_provider) } diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 84a0e8c3ea..ebd0ce363f 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -160,6 +160,34 @@ impl UserDraftItemKind { } } + /// The `draft.value` key holding the user-typed target path — where a + /// deploy of this draft would land when the user staged a rename. A script + /// draft round-trips its own `path`; flows and apps write a separate + /// `draft_path`, and only when it differs from the row's path. + /// + /// `None` for every kind with no editor to stage a rename in — a resource, + /// a variable, a trigger. Their drafts carry no such key, so `Some` is also + /// the test for whether a kind can be moved at all. + pub fn typed_path_field(&self) -> Option<&'static str> { + use UserDraftItemKind::*; + match self { + Script => Some("path"), + Flow | App | RawApp => Some("draft_path"), + _ => None, + } + } + + /// The other of the two path keys a draft value can carry. The editors mirror + /// the typed path into it while it differs from the row's path, so a move that + /// rewrote only `typed_path_field` would leave the mirror naming the old + /// location — and the loaders prefer the mirror, which un-does the move. + pub fn mirror_path_field(&self) -> Option<&'static str> { + match self.typed_path_field()? { + "path" => Some("draft_path"), + _ => Some("path"), + } + } + /// Whether OTHER users' drafts at a path are visible to a viewer (the /// "others are editing" list, owner circles, and the `get_draft_for_user` /// View JSON / Fork endpoint). Enabled only for the full-page editor items @@ -212,6 +240,12 @@ pub struct WithDraftOverlay { pub is_draft: bool, #[serde(skip_serializing_if = "Option::is_none")] pub draft_saved_at: Option>, + /// The deployed version the draft forked from (`draft.base`), as text + /// whatever the kind. The editor compares it to the head it loaded to tell + /// a draft that is behind. Absent when there is no draft or it was never + /// forked from a deploy. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_base: Option, /// True when no deployed row exists at this path: `inner` is only a /// best-effort stand-in synthesized from the draft and only `draft` is /// canonical. Frontend uses this to disable "diff/reset vs deployed" and @@ -245,7 +279,9 @@ async fn fetch_other_drafts_users( // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled. Else a real teammate's draft renders // as a phantom "Legacy draft". The genuine NULL-email legacy row keeps - // `username = None` (no `usr`/`password` match and `d.email` is NULL). + // `username = None` (no `usr`/`password` match and `d.email` is NULL), which is + // why an owner that resolves to no name at all — an external JWT's subject has + // neither row — is dropped instead: `None` is taken to mean "legacy" downstream. let rows = sqlx::query_as!( OtherDraftUser, r#"SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as "username?", @@ -261,6 +297,7 @@ async fn fetch_other_drafts_users( AND d.path = $2 AND d.typ = $3 AND (d.email IS NULL OR d.email <> $4) + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ORDER BY d.email NULLS LAST"#, w_id, path, @@ -294,6 +331,7 @@ where inner: Box::new(deployed), is_draft: false, draft_saved_at: None, + draft_base: None, no_deployed: false, draft: None, other_drafts_users: Vec::new(), @@ -314,7 +352,7 @@ where // row when an owned one exists. let row = sqlx::query!( r#"SELECT value as "value!: sqlx::types::Json>", - created_at + created_at, base FROM draft WHERE workspace_id = $1 AND (email = $2 OR email IS NULL) @@ -335,6 +373,7 @@ where inner: Box::new(deployed), is_draft: false, draft_saved_at: None, + draft_base: None, no_deployed: false, draft: None, other_drafts_users, @@ -347,6 +386,7 @@ where inner: Box::new(deployed), is_draft: true, draft_saved_at: Some(row.created_at), + draft_base: row.base, no_deployed: false, draft: Some(draft_json), other_drafts_users, @@ -398,6 +438,45 @@ pub async fn fetch_draft_only_list_rows( Ok(rows) } +/// Delete the caller's OWN draft at a path with no deployed row, for the DELETE +/// route of a kind whose list synthesizes such rows via +/// `fetch_draft_only_list_rows`. The `NOT EXISTS` leaves a deployed row's draft +/// alone, so a route may call this on its not-found branch whatever the reason +/// for the miss. `Ok(false)` means nothing matched: the caller reports its own error. +/// +/// Takes no permission check and callers must not add one: an email-scoped row +/// belongs to the caller, who can always discard it, as `update_draft`'s +/// own-discard does. Legacy (`email IS NULL`) rows are owned by nobody and keep +/// their write gate, so discarding one stays on the `update_draft` route. +pub async fn delete_draft_only_for_path( + db: &DB, + w_id: &str, + kind: UserDraftItemKind, + path: &str, + email: &str, +) -> Result { + let Some(table) = kind.deployed_table() else { + return Ok(false); + }; + // `table` is from the closed `deployed_table()` enum, never user input. + let sql = format!( + "DELETE FROM draft \ + WHERE workspace_id = $1 AND typ = $2::text::DRAFT_KIND AND path = $3 \ + AND email = $4 \ + AND NOT EXISTS (SELECT 1 FROM {table} t \ + WHERE t.workspace_id = draft.workspace_id AND t.path = draft.path)" + ); + let deleted = sqlx::query(&sql) + .bind(w_id) + .bind(kind.as_str()) + .bind(path) + .bind(email) + .execute(db) + .await? + .rows_affected(); + Ok(deleted > 0) +} + /// The get-by-path draft choreography, shared by every entity's "get by path" /// route. Given the deployed entity as an `Option` (caller maps its own "not /// found" to `None`): @@ -426,6 +505,93 @@ pub async fn overlay_or_draft_only( } } +/// Delete the drafts an address owns, and the move records routing saves to them, +/// across every workspace. +/// +/// `draft.email` carries no foreign key to `password`: a draft's owner is any principal the +/// instance authenticates, and an external JWT's subject never has a `password` row. Deleting an +/// account is therefore what has to delete its drafts — a delete path that skips this leaves them +/// behind forever, addressed to someone who no longer exists. Call it in the same transaction as +/// the account removal. +/// +/// No authorization of its own: it acts instance-wide on whatever address it is handed, so the +/// caller must already have authorized removing that account (superadmin, the account's own +/// holder, or SCIM). +pub async fn delete_drafts_of_email(conn: &mut sqlx::PgConnection, email: &str) -> Result<()> { + sqlx::query!("DELETE FROM draft WHERE email = $1", email) + .execute(&mut *conn) + .await?; + // The records routing saves to those drafts go with them: one left behind would send + // a later holder of the address to a draft that no longer exists. + sqlx::query!("DELETE FROM draft_move WHERE email = $1", email) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Move the drafts an address owns onto its new address, for the same reason +/// [`delete_drafts_of_email`] exists: no foreign key follows the rename, so drafts left behind are +/// stranded on an address that no longer authenticates. Same authorization contract, for a rename. +/// +/// The two addresses may each already hold a draft of the same item, since the destination can +/// belong to a principal with no account and so is not covered by the caller's "address is free" +/// check. `draft_pkey_with_user` admits only one, so the moving account's wins — which is also why +/// a rename onto the same address returns early: every row would collide with itself and be +/// cleared. Callers need not compare first (an IdP re-sending an unchanged `userName` does not). +pub async fn rename_drafts_of_email( + conn: &mut sqlx::PgConnection, + old_email: &str, + new_email: &str, +) -> Result<()> { + if old_email == new_email { + return Ok(()); + } + sqlx::query!( + "DELETE FROM draft dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.path = dest.path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + // The records that route saves to those drafts follow the same address, or a save + // still addressed to a path the account moved away from would start a second draft + // there. The moving account's record for a path wins, as its draft just did. + sqlx::query!( + "DELETE FROM draft_move dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft_move src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.old_path = dest.old_path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft_move SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Delete EVERY user's draft (and the legacy NULL-email row) at a path+kind. /// Use when the item is DELETED outright: it's gone for everyone, so leaving /// teammates' drafts behind would orphan them forever. Discarding one's OWN @@ -479,6 +645,247 @@ pub async fn delete_own_draft_for_path( Ok(()) } +/// Carry every draft at `old_path` over to `new_path` when an item MOVES +/// (rename or relocation). A draft is bound to its item by nothing but the path +/// string, so without this a move detaches every draft on the item. No owner +/// filter: teammates' rows and the legacy NULL-email row follow too. +/// +/// **The caller must have authorized the underlying item move first.** This rewrites +/// rows the caller has no permission on and enforces nothing itself, taking both paths +/// on trust: it is safe only inside a deploy that has already cleared them, and is a +/// cross-user write with no gate anywhere else. +/// +/// The value keeps its base version, so every carried draft reads as behind the head the +/// move minted. Of its two path keys, one still naming `old_path` follows the row (the +/// editors write the item's own path there on every save, so it is the item's path and +/// not a staged rename); any other value is a rename the user staged, and is kept. +/// +/// A draft already at `new_path` occupies it the way a deployed item does, so the move is +/// refused with `BadRequest` inside the deploy's transaction, refusing the rename itself: +/// moving onto it would merge two items or strand the row that lost. A row a `draft_move` +/// record already brought there from `old_path` is this rename's own draft and does not +/// count: deploying it would otherwise be refused against itself, with no way out. +pub async fn move_drafts_for_path( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + old_path: &str, + new_path: &str, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + // Named by workspace username, as the editors name other users' drafts: the + // caller is often not the owner, and cannot clear a draft they cannot find. + let owners = sqlx::query!( + r#"SELECT d.email IS NULL as "legacy!", COALESCE(u.username, p.username) as username + FROM draft d + LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true + WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ::text = ANY($3::text[]) + -- A row a move already brought here from the very path being renamed is this + -- rename's own draft, not an item in its way: the record says which owners + -- it moved (all of them for an item move, one for a draft-only move). Any + -- other row is a second item and still collides. + AND NOT EXISTS ( + SELECT 1 FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ::text = ANY($3::text[]) + AND m.old_path = $4 AND m.new_path = $2 + AND (m.email IS NULL OR m.email IS NOT DISTINCT FROM d.email) + ) + ORDER BY 2"#, + w_id, + new_path, + &typs as &[&str], + old_path, + ) + .fetch_all(&mut **tx) + .await?; + if !owners.is_empty() { + let names = owners + .into_iter() + .map(|o| match (o.username, o.legacy) { + (Some(name), _) => name, + (None, true) => "a legacy workspace draft".to_string(), + (None, false) => "another user".to_string(), + }) + .collect::>() + .join(", "); + return Err(crate::error::Error::BadRequest(format!( + "'{new_path}' already has a draft on it ({names}) — it must be moved or discarded first" + ))); + } + // `draft.value` is `json`, so a NUL escape left by a pre-sanitizer write makes + // `to_jsonb` raise 22P05. `clean` is `strip_json_nul`'s parity rule in SQL (escaped + // backslashes parked on chr(1), which a `json` value's text cannot hold, so nothing + // collides with it; chr(92) spells the backslash so no escape sequence reaches this + // file). A row whose keys need re-pointing is therefore rewritten clean rather than + // left naming the old path, and one that needs nothing keeps its value byte for byte. + sqlx::query!( + r#"UPDATE draft AS d + SET path = $3::text, + value = ( + SELECT CASE + WHEN s.clean -> 'path' = to_jsonb($2::text) + OR s.clean -> 'draft_path' = to_jsonb($2::text) + THEN to_json( + s.clean + || CASE WHEN s.clean -> 'path' = to_jsonb($2::text) + THEN jsonb_build_object('path', $3::text) + ELSE '{}'::jsonb END + || CASE WHEN s.clean -> 'draft_path' = to_jsonb($2::text) + THEN jsonb_build_object('draft_path', $3::text) + ELSE '{}'::jsonb END + ) + ELSE d.value + END + FROM (SELECT replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), + chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean) s + ) + WHERE d.workspace_id = $1 + AND d.path = $2::text + AND d.typ::text = ANY($4::text[])"#, + w_id, + old_path, + new_path, + &typs as &[&str], + ) + .execute(&mut **tx) + .await?; + record_draft_move(tx, w_id, kinds, old_path, new_path, None).await +} + +/// Record that the drafts at `old_path` now live at `new_path`, so a draft save still +/// addressed to `old_path` lands on them (see `update_draft`). `email` scopes the +/// record to one user's draft-only move; `None` is a deployed item's move, for everyone. +/// +/// Kept to one hop: records pointing at `old_path` are re-pointed (an owner's move +/// leaves the item's own record alone and copies it into its own scope instead), and +/// records leaving either path are replaced, since `new_path` now holds the item. +/// +/// **The caller must have authorized the move first.** A record routes every later +/// draft write at `old_path` (any owner's, for an item move), and enforces nothing +/// itself. +/// +/// A record outlives the editors that need it: it ends when a later move touches either +/// path or an item is deployed at `old_path`, so a save meaning to start a NEW draft at a +/// vacated path would be routed instead. Nothing does that today: every surface parks a +/// new item at a minted `u//draft_` key (`mintDraftPath.ts`). +pub async fn record_draft_move( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + old_path: &str, + new_path: &str, + email: Option<&str>, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + sqlx::query!( + "DELETE FROM draft_move + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND old_path IN ($3, $4) AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + // Routes that ended at the destination before this move describe drafts that were + // carried there for an item this one is replacing: left alive, a save addressed to + // the start of that chain would land on this move's draft instead. + sqlx::query!( + "DELETE FROM draft_move + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND new_path = $4 AND old_path <> $3 AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE draft_move SET new_path = $4 + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND new_path = $3 AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + // An owner's move must not re-point what everyone else follows, so the item's own + // records ending at `old_path` are copied into this scope: a save addressed to the + // start of that chain still reaches this destination in one hop. Only those: another + // user's owner-scoped record routes their own draft-only item, which this move has + // nothing to do with. + if email.is_some() { + sqlx::query!( + "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email) + SELECT m.workspace_id, m.typ, m.old_path, $4, $5::text + FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ::text = ANY($2::text[]) + AND m.new_path = $3 AND m.email IS NULL + AND NOT EXISTS ( + SELECT 1 FROM draft_move o + WHERE o.workspace_id = m.workspace_id AND o.typ = m.typ + AND o.old_path = m.old_path AND o.email = $5::text + )", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + } + sqlx::query!( + "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email) + SELECT $1, t::draft_kind, $3, $4, $5 FROM unnest($2::text[]) t", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Drop the move records at `path`: an item was just created there, so saves addressed to +/// it are its own (records leaving `path`). A deploy that is not a rename also drops the +/// records arriving, which point at an item that no longer owns the path; `keep_from`, +/// the path a rename came from, suppresses that, since the chain ending here is the one +/// this very deploy just wrote. +/// +/// **The caller must have authorized the deploy that created the item first.** +/// Dropping a record sends later draft writes at `path` back to `path`. +pub async fn clear_draft_moves_from( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + path: &str, + keep_from: Option<&str>, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + sqlx::query!( + "DELETE FROM draft_move WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND (old_path = $3 OR (new_path = $3 AND $4::text IS NULL))", + w_id, + &typs as &[&str], + path, + keep_from, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + /// Fetch the authed user's draft as a standalone payload, for "get by path" /// routes when no deployed row exists but a draft might. Returns it as a /// `WithDraftOverlay` with `inner` and `draft` both set to the same JSON and @@ -497,7 +904,7 @@ pub async fn fetch_draft_only( // Own draft first, legacy NULL-email row as fallback (see `maybe_overlay_draft`). let row = sqlx::query!( r#"SELECT value as "value!: sqlx::types::Json>", - created_at + created_at, base FROM draft WHERE workspace_id = $1 AND (email = $2 OR email IS NULL) @@ -528,6 +935,7 @@ pub async fn fetch_draft_only( inner: Box::new(draft_json.clone()), is_draft: true, draft_saved_at: Some(row.created_at), + draft_base: row.base, no_deployed: true, draft: Some(draft_json), other_drafts_users, diff --git a/backend/windmill-common/src/usernames.rs b/backend/windmill-common/src/usernames.rs index 6fde2a8186..adb81ae682 100644 --- a/backend/windmill-common/src/usernames.rs +++ b/backend/windmill-common/src/usernames.rs @@ -17,6 +17,23 @@ lazy_static::lazy_static! { pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); } +/// Width of the `username` columns of `usr`, `password` and `pending_user`. +pub const USERNAME_MAX_LEN: usize = 50; + +/// `base` with the collision suffix of `attempt` appended (none for the first attempt), cut +/// to `USERNAME_MAX_LEN`. A local part longer than the column is a valid email, and an +/// insert that fails on the derived username rolls back everything around it. +pub fn fit_username(base: &str, attempt: u32) -> String { + let suffix = if attempt > 1 { + attempt.to_string() + } else { + String::new() + }; + let mut username: String = base.chars().take(USERNAME_MAX_LEN - suffix.len()).collect(); + username.push_str(&suffix); + username +} + pub async fn generate_instance_wide_unique_username<'c>( tx: &mut Transaction<'c, Postgres>, email: &str, @@ -41,9 +58,7 @@ pub async fn generate_instance_wide_unique_username<'c>( email ))); } - if i > 1 { - username = format!("{}{}", base_username, i) - } + username = fit_username(&base_username, i); username_conflict = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", &username, @@ -164,3 +179,19 @@ pub async fn get_instance_username_or_create_pending<'c>( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fit_username_keeps_the_column_width() { + assert_eq!(fit_username("alice", 1), "alice"); + assert_eq!(fit_username("alice", 2), "alice2"); + let base = "a".repeat(60); + assert_eq!(fit_username(&base, 1), "a".repeat(USERNAME_MAX_LEN)); + let with_suffix = fit_username(&base, 1000); + assert_eq!(with_suffix.len(), USERNAME_MAX_LEN); + assert!(with_suffix.ends_with("1000")); + } +} diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 925fe32a4a..1b696746b0 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -13,6 +13,35 @@ lazy_static::lazy_static! { pub static ref VALID_EMAIL: regex::Regex = regex::Regex::new( r"^[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?\.)+[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$" ).unwrap(); + +} + +/// Width of the `email` columns of `usr`, `workspace_invite` and `email_to_igroup`. +pub const EMAIL_COLUMN_MAX_LEN: usize = 255; + +/// The regex of the `proper_email` CHECK constraint on `usr` and `workspace_invite` +/// (`20220620210708_regex_fix`), verbatim, for [`usr_accepts_email`]. Evaluated by the +/// database and never by a Rust engine: `~*` folds case under the database collation, so a +/// fixed mirror accepts addresses the constraint rejects, or rejects ones it holds, on some +/// locale. `windmill-common/tests/usr_accepts_email.rs` pins the text to the constraint. +pub const PROPER_EMAIL_PATTERN: &str = r#"^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$"#; + +/// Whether `usr` (and `workspace_invite`) will store `email`: the `proper_email` regex as the +/// database evaluates it, plus the column width. Unlike [`VALID_EMAIL`] this admits every +/// address those tables already hold, which matters wherever an existing member is judged. +pub async fn usr_accepts_email<'c, E>(db: E, email: &str) -> crate::error::Result +where + E: sqlx::Executor<'c, Database = sqlx::Postgres>, +{ + if email.contains('\0') || email.chars().count() > EMAIL_COLUMN_MAX_LEN { + return Ok(false); + } + let accepted: bool = sqlx::query_scalar("SELECT $1::text ~* $2::text") + .bind(email) + .bind(PROPER_EMAIL_PATTERN) + .fetch_one(db) + .await?; + Ok(accepted) } pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev"; @@ -21,6 +50,13 @@ pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev"; pub const COOKIE_NAME: &str = "token"; +/// `password.login_type` of an account created for someone before they have signed in: +/// no credential of its own (password login and reset require `'password'`), reachable +/// only through a superadmin-minted login link until either `set_password` turns it into +/// a password account or the first OAuth login proving the same address adopts it and +/// rewrites `login_type` to the provider. +pub const PENDING_OAUTH_LOGIN_TYPE: &str = "pending_oauth"; + /// Prefix for user-based permissioned_as values: "u/" pub const PERMISSIONED_AS_USER_PREFIX: &str = "u/"; /// Prefix for group-based permissioned_as values: "g/" @@ -31,6 +67,29 @@ pub const USERNAME_GROUP_PREFIX: &str = "group-"; /// columns runnables and triggers store one in. pub const PERMISSIONED_AS_MAX_LEN: usize = 55; +/// Whether any account exists for `email`: a `password` row (deactivated ones +/// included, since the sign-in path filters `disabled = false` and a re-enabled +/// account must not read as absent) or a `usr` row in any workspace (what a service +/// account has instead of a password). A guest is someone with none: the single rule +/// that keeps an account holder from ever holding a cheaper guest identity. +/// +/// The address is lowercased before the lookup: accounts are stored lowercased, so a +/// mixed-case address would otherwise miss an existing account and be let through. The +/// comparison stays a plain equality (not `lower(email)`), so it uses the email index. +pub async fn has_any_account<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, + email: &str, +) -> crate::error::Result { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1) + OR EXISTS(SELECT 1 FROM usr WHERE email = $1)", + ) + .bind(email.to_lowercase()) + .fetch_one(executor) + .await + .map_err(|e| crate::error::Error::internal_err(format!("checking account for {email}: {e:#}"))) +} + /// An email-shaped username is its own principal, which is how a superadmin acting without a /// `usr` row is named (`usr.username` is constrained to `[\w-]+`, so a member never is). It is /// decided before the group convention — an address is never a group's username — and one @@ -152,13 +211,19 @@ pub async fn permissioned_as_exists( /// Drop a cached address so a transactional email change is visible immediately. /// -/// The address is derived at dispatch and feeds the instance-superadmin check and -/// `email_to_igroup`, so serving a stale one would run jobs with the wrong authorization -/// for up to the cache TTL. +/// Not the thing that keeps authorization correct — `fetch_authed_from_permissioned_as` +/// re-resolves the address before granting anything. This keeps the cache from serving an +/// address that is merely wrong for the TTL, on reads and on what is shown. pub fn invalidate_email_cache(workspace_id: &str, username: &str) { EMAIL_CACHE.remove(&(workspace_id.to_string(), username.to_string())); } +/// Drop this name's entry in every workspace, for the changes that know the name but not the +/// workspace: a superadmin resolves through `password`, whose row names no workspace of its own. +pub fn invalidate_email_cache_for_username(username: &str) { + EMAIL_CACHE.retain(|(_workspace_id, cached_username), _| cached_username != username); +} + /// Inverse of [`get_email_from_permissioned_as`]: the principal an on-behalf-of email /// names in this workspace, for callers that supply the email alone. /// @@ -171,6 +236,14 @@ pub fn invalidate_email_cache(workspace_id: &str, username: &str) { /// not a superadmin's, or a group that no longer exists. Callers then leave the identity /// unrecorded rather than storing a principal that cannot authenticate. /// +/// Known, accepted consequence of a real account winning the synthetic `group-*@windmill.dev` +/// namespace: a group identity sent as its address alone, as a "keep target identity" workspace +/// deploy sends it for scripts, flows and apps, comes back as the account holding that address +/// when one exists, not as `g/*`. Such an account takes an admin to exist: a superadmin or an +/// admin-configured identity provider to create it (the public OAuth providers only assert a +/// `@windmill.dev` address to that domain's owner) and an admin of the target workspace to admit +/// it, so no member can steer a group's runnables to themselves this way. +/// /// Reads through the non-RLS pool and authorizes nothing: callers must already be authorized /// for `workspace_id`. pub async fn permissioned_as_from_email( @@ -219,6 +292,33 @@ pub async fn permissioned_as_from_email( /// - "u/{username}" → resolve via [`resolve_username_to_email`] (cached) /// - "g/{group}" → "group-{group}@windmill.dev" /// - raw email → return as-is +/// +/// `notify_user_email_change` evicts the key on every process for each change that can move it, +/// at that process's next notify-event poll (`LISTEN_NEW_EVENTS_INTERVAL_SEC`, 10s by default), +/// so a hit can still be the old address for up to one poll. The TTL caps it if an eviction is +/// ever missed. +/// +/// Which of the two to use is a question of how long a wrong answer lives, not of whether it is +/// stored — both of these get stored and read back. A config row (an app policy, a schedule, a +/// runnable) is the authority for every run that follows it, so a stale address there is +/// permanent and invisible: those use [`get_email_from_permissioned_as_uncached`]. Job dispatch +/// also stores its answer, and the worker reads it back to build that run's authed, but it +/// governs one job and dies with it, so it stays here. +/// +/// The job's own authorization does not trust the address as given: +/// `fetch_authed_from_permissioned_as` re-resolves it from the principal's live binding, and that +/// corrected address is what the job row and its token carry. Route an address into an `Authed`, +/// a job row or a token without going through that function, and this cache stops being safe to +/// read at dispatch. +/// +/// What reads the dispatch address before that re-resolution (the quota and superadmin-exemption +/// checks at the top of `push_inner`, a flow step's tag check) or when the principal has no live +/// binding can act on the old address for up to one poll after a username reuse, an email change +/// or a superadmin change. That window is accepted as the cost of keeping dispatch off the +/// database; a consumer that cannot tolerate it must re-resolve first. +/// +/// Reads through the non-RLS pool and authorizes nothing — callers must already be authorized +/// for `workspace_id`. pub async fn get_email_from_permissioned_as<'c>( permissioned_as: &str, workspace_id: &str, @@ -227,13 +327,21 @@ pub async fn get_email_from_permissioned_as<'c>( get_email_from_permissioned_as_inner(permissioned_as, workspace_id, db, true).await } -/// [`get_email_from_permissioned_as`] without the address cache. Nothing evicts that cache -/// across processes, so for a minute after an email change it still serves the old address — -/// fine where the address only labels something on screen, wrong where it decides whether a -/// write is accepted or is copied onto a job row that outlives the window. +/// [`get_email_from_permissioned_as`] for a value about to be **persisted**. /// -/// Reads through the non-RLS pool and authorizes nothing, like the cached one: callers must -/// already be authorized for `workspace_id`. +/// The eviction is delivered by the `notify_event` poller, not synchronously, so for a few +/// seconds after a change a replica can still serve the old address. In a config row that is +/// permanent: the row outlives the eviction, every later run trusts it, and nothing re-derives +/// it, so a principal and an address that name different accounts stay that way. +/// +/// Use this for three cases, all of which end in a stored pair: +/// - writing the address into a row; +/// - the lookup that validates a pair before it is stored; +/// - **reads whose result the client sends back** — a script or a workspace export hands over a +/// principal and address together, and a redeploy validates that pair against a fresh +/// resolution, so a stale one comes back as a rejected deploy rather than a stale display. +/// +/// See [`get_email_from_permissioned_as`] for the dispatch case that deliberately does not. pub async fn get_email_from_permissioned_as_uncached<'c>( permissioned_as: &str, workspace_id: &str, diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 62d47a34d7..29fc9e99f3 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -479,6 +479,28 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) { (per_page, offset) } +/// [`paginate`] for a listing that answers whole unless a size is asked for: the deploy +/// histories, which the history panels and the CLI read unpaged while the diff picker takes +/// a page at a time. An asked-for size is still clamped, and the offset saturates rather +/// than wrapping, so no caller can turn this into an unbounded scan or a negative bind. +pub fn paginate_optional(pagination: Pagination) -> (i64, i64) { + // Naming neither parameter asks for the whole listing, the contract these endpoints + // have always answered on. Naming either makes it a page like any other listing's. + if pagination.page.is_none() && pagination.per_page.is_none() { + return (i64::MAX, 0); + } + let per_page = pagination + .per_page + .unwrap_or(DEFAULT_PER_PAGE) + .clamp(1, MAX_PER_PAGE) as i64; + // Bound before Postgres sees it: an unchecked cast of a caller-controlled page becomes + // a negative OFFSET, which is an error rather than an empty page. + let offset = i64::try_from(pagination.page.unwrap_or(1).max(1) - 1) + .unwrap_or(i64::MAX) + .saturating_mul(per_page); + (per_page, offset) +} + pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) { let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE); let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; @@ -976,18 +998,33 @@ fn six_fields_hint(schedule_str: &str, version: Option<&str>, seconds_required: } impl ScheduleType { + /// `NotFound` means the expression has no run left (an expired year, an impossible + /// date), and schedule pushes disable the schedule on it. Every other error must stay + /// transient: croner fails across a DST jump longer than an hour (Antarctica/Troll) + /// and succeeds again once the jump has passed. pub fn find_next( &self, starting_from: &chrono::DateTime, - ) -> chrono::DateTime { + ) -> Result> { + let no_run_left = || { + Error::NotFound(format!( + "cron: the schedule has no run left after {}", + starting_from.format("%Y-%m-%d %H:%M:%S %Z") + )) + }; match self { ScheduleType::Croner(croner_schedule) => croner_schedule .find_next_occurrence(starting_from, false) - .expect("cron: a schedule should have a next event"), - ScheduleType::Cron(schedule) => schedule - .after(starting_from) - .next() - .expect("cron: a schedule should have a next event"), + .map_err(|e| match e { + croner::errors::CronError::TimeSearchLimitExceeded => no_run_left(), + e => Error::internal_err(format!( + "cron: could not compute the run after {}: {e}", + starting_from.format("%Y-%m-%d %H:%M:%S %Z") + )), + }), + ScheduleType::Cron(schedule) => { + schedule.after(starting_from).next().ok_or_else(no_run_left) + } } } @@ -1125,19 +1162,35 @@ use tokio::time::{self, Duration, Sleep}; use pin_project_lite::pin_project; +/// What a [`WarnAfterFuture`] is timing, which decides how its warning reads. +pub enum WarnSubject { + /// A database query, with the SQL when the caller has it. + Query(Option), + /// Anything else (a child process, a cache transfer), named for the log line. + Step(String), +} + pub trait WarnAfterExt: Future + Sized { /// Warns if the future takes longer than the specified number of seconds to complete. #[track_caller] fn warn_after_seconds(self, seconds: u8) -> WarnAfterFuture { let caller = Location::caller(); - self.build_from_caller(seconds, caller, None) + self.build_from_caller(seconds, caller, WarnSubject::Query(None)) + } + + /// Same, for a step that is not a database query (a child process, a cache transfer): + /// the warning names `step` instead of reporting a slow query. + #[track_caller] + fn warn_after_seconds_for(self, seconds: u8, step: &str) -> WarnAfterFuture { + let caller = Location::caller(); + self.build_from_caller(seconds, caller, WarnSubject::Step(step.to_string())) } fn build_from_caller( self, seconds: u8, caller: &Location, - sql: Option, + subject: WarnSubject, ) -> WarnAfterFuture { let location = format!("{}:{}", caller.file(), caller.line()); WarnAfterFuture { @@ -1147,13 +1200,13 @@ pub trait WarnAfterExt: Future + Sized { start_time: std::time::Instant::now(), location, seconds, - sql, + subject, } } #[track_caller] fn warn_after_seconds_with_sql(self, seconds: u8, sql: String) -> WarnAfterFuture { let caller = Location::caller(); - self.build_from_caller(seconds, caller, Some(sql)) + self.build_from_caller(seconds, caller, WarnSubject::Query(Some(sql))) } } @@ -1171,7 +1224,7 @@ pin_project! { location: String, start_time: std::time::Instant, seconds: u8, - sql: Option, + subject: WarnSubject, } } @@ -1191,12 +1244,20 @@ impl Future for WarnAfterFuture { // Poll the timeout future to check if it has elapsed. if !*this.warned { if this.timeout.poll(cx).is_ready() { - tracing::warn!( - location = this.location, - "SLOW_QUERY: query {} to db taking longer than expected (> {} seconds)", - build_query_string(&this.location, this.sql.as_deref()), - this.seconds, - ); + match &*this.subject { + WarnSubject::Step(step) => tracing::warn!( + location = this.location, + "SLOW_STEP: {step} at {} taking longer than expected (> {} seconds)", + this.location, + this.seconds, + ), + WarnSubject::Query(sql) => tracing::warn!( + location = this.location, + "SLOW_QUERY: query {} to db taking longer than expected (> {} seconds)", + build_query_string(&this.location, sql.as_deref()), + this.seconds, + ), + } *this.warned = true; } } @@ -1206,12 +1267,20 @@ impl Future for WarnAfterFuture { Poll::Ready(output) => { if *this.warned { let elapsed = this.start_time.elapsed(); - tracing::warn!( - location = this.location, - "SLOW_QUERY: completed query {} with total duration: {:.2?}", - build_query_string(&this.location, this.sql.as_deref()), - elapsed - ); + match &*this.subject { + WarnSubject::Step(step) => tracing::warn!( + location = this.location, + "SLOW_STEP: {step} at {} completed with total duration: {:.2?}", + this.location, + elapsed + ), + WarnSubject::Query(sql) => tracing::warn!( + location = this.location, + "SLOW_QUERY: completed query {} with total duration: {:.2?}", + build_query_string(&this.location, sql.as_deref()), + elapsed + ), + } } Poll::Ready(output) } @@ -1625,6 +1694,31 @@ pub fn truncate_with_ellipsis(s: &str, max_chars: usize) -> String { mod tests { use super::*; + #[test] + fn test_paginate_optional_answers_whole_but_bounds_what_is_asked_for() { + // Nothing asked for: every row there can be, which is what the history panels and + // the CLI read. + assert_eq!( + paginate_optional(Pagination { page: None, per_page: None }), + (i64::MAX, 0) + ); + assert_eq!( + paginate_optional(Pagination { page: Some(3), per_page: Some(20) }), + (20, 40) + ); + // An asked-for size is still capped, so no caller turns this into an unbounded scan. + assert_eq!( + paginate_optional(Pagination { page: None, per_page: Some(usize::MAX) }), + (MAX_PER_PAGE as i64, 0) + ); + // A page nobody could mean lands past the end rather than going negative, which + // Postgres would reject outright. + let (per_page, offset) = + paginate_optional(Pagination { page: Some(usize::MAX), per_page: Some(20) }); + assert_eq!(per_page, 20); + assert_eq!(offset, i64::MAX); + } + /// A 5-field crontab line is the most common way to get a schedule rejected, and both /// parsers report it in terms a crontab user cannot act on, so the seconds field and the /// equivalent expression must reach the caller for v1 and v2 alike. @@ -1677,6 +1771,22 @@ mod tests { assert!(!err.contains("6 fields"), "{err}"); } + #[test] + fn find_next_reports_only_a_cron_with_no_run_left_as_not_found() { + use chrono::TimeZone; + let troll: chrono_tz::Tz = "Antarctica/Troll".parse().unwrap(); + // Troll's clocks jump from 01:00 to 03:00 on the last Sunday of March. + let before_jump = troll.with_ymd_and_hms(2027, 3, 28, 0, 30, 0).unwrap(); + + let expired = ScheduleType::from_str("0 0 9 1 1 * 2026", Some("v1"), true).unwrap(); + let err = expired.find_next(&before_jump).unwrap_err(); + assert!(matches!(err, Error::NotFound(_)), "{err}"); + + let across_jump = ScheduleType::from_str("0 30 1 * * *", Some("v2"), true).unwrap(); + let err = across_jump.find_next(&before_jump).unwrap_err(); + assert!(!matches!(err, Error::NotFound(_)), "{err}"); + } + /// A worker that restarts must land on the exact same name to reclaim its `worker_ping` /// row, while still never colliding with the other workers of its own process. The /// suffix must also stay a single `-` segment, which is what the interactive shell tag diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 2d9e1d2504..c5dbe06fcd 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -259,12 +259,17 @@ pub async fn build_crypt_with_key_suffix( key_suffix: &str, ) -> crate::error::Result { let key = get_workspace_key(w_id, db).await?; + Ok(crypt_from_key_with_suffix(&key, key_suffix)) +} + +/// The cipher `build_crypt_with_key_suffix` builds, from a key string in hand. +pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> MagicCrypt256 { let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { format!("{}{}{}", key, salt, key_suffix) } else { format!("{}{}", key, key_suffix) }; - Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) + magic_crypt::new_magic_crypt!(crypt_key, 256) } pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result { diff --git a/backend/windmill-common/src/wac.rs b/backend/windmill-common/src/wac.rs index d46c937e33..05c343b427 100644 --- a/backend/windmill-common/src/wac.rs +++ b/backend/windmill-common/src/wac.rs @@ -8,11 +8,17 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::{Postgres, Transaction}; +use std::sync::atomic::AtomicBool; use uuid::Uuid; use crate::error::{self, Error}; use crate::DB; +/// Set when a child completion brought a parked WAC parent's `suspend` counter to +/// zero, so a worker's pull loop tries the suspended-jobs query first instead of +/// waiting for its next periodic attempt. +pub static WAC_SUSPEND_READY: AtomicBool = AtomicBool::new(false); + /// Checkpoint state persisted across workflow invocations. #[derive(Debug, Serialize, Deserialize, Default, Clone)] pub struct WacCheckpoint { @@ -672,3 +678,199 @@ pub async fn persist_inline_checkpoint_delta( Ok(failure) } + +/// The step key a WAC v2 parent is waiting on `child_job` for, if any. +/// +/// `job_ids` is the parent's `pending_steps.job_ids` (step key → child job id). +/// A child absent from it is not a step this round is waiting on: a completion +/// arriving after the parent re-dispatched the key to a new job, or a child the +/// body launched directly (`runScript` and friends). +fn pending_step_key(job_ids: &Value, child_job: &Uuid) -> Option { + let child = child_job.to_string(); + job_ids + .as_object()? + .iter() + .find_map(|(key, id)| (id.as_str() == Some(child.as_str())).then(|| key.clone())) +} + +/// Record a completed child on its WAC parent, in the child's completion transaction. +/// +/// Every path that brings a child to a terminal state — a worker's result, the +/// zombie monitor, a cancel — completes it through `add_completed_job`, and this is +/// where the parent learns of it. For a child the parent is parked on, the step's +/// result (or failure record) is merged into the checkpoint's `completed_steps` and +/// the parent's `suspend` counter drops by one, atomically with the child's own +/// completion: there is no window in which the child is completed but the parent +/// still waits for it. For any other child, only the timeline entry is stamped. +/// +/// Returns whether the counter reached zero, i.e. the parent is ready to be pulled. +/// +/// Exactly once: the merge is refused when `completed_steps` already holds the key +/// or `job_ids` no longer maps it to this child, and the decrement follows only a +/// merge that happened. Two completions of one child (a worker and the monitor +/// racing) therefore decrement once, and a stale completion never touches a +/// counter that belongs to a later round. +/// +/// Lock order: the parent's queue row, then its status row, then (by the caller) +/// the child's queue row. A cancel walks parent then children, the parent's own +/// completion deletes its queue row and cascades to its status row, and the park +/// (`suspend_wac_parent`) locks the queue row before writing the checkpoint, so +/// any other order can deadlock against one of them. +/// +/// Authorization: none is checked here. `child_job` is the job the caller is +/// completing, which it already holds, and `parent_job` must be that job's +/// persisted `v2_job.parent_job` (both callers read it from the child's row). +/// The read that gates the step merge and the counter decrement joins on that +/// relationship, so a mismatched pair changes no parent's `completed_steps` or +/// `suspend`; only the timeline stamp at the end runs unconditionally. Job ids +/// are global, so no workspace scoping is needed on top. +pub async fn record_child_completion( + tx: &mut Transaction<'_, Postgres>, + parent_job: &Uuid, + child_job: &Uuid, + success: bool, + duration_ms: i64, + result: &str, +) -> error::Result { + let job_ids: Option> = sqlx::query_scalar( + "SELECT s.workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ + FROM v2_job_status s JOIN v2_job c ON c.parent_job = s.id \ + WHERE s.id = $1 AND c.id = $2", + ) + .bind(parent_job) + .bind(child_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to read WAC parent {parent_job}: {e}")))?; + + let step_key = job_ids + .flatten() + .and_then(|ids| pending_step_key(&ids, child_job)); + + let mut parent_ready = false; + if let Some(step_key) = step_key { + let parked: Option = + sqlx::query_scalar("SELECT suspend FROM v2_job_queue WHERE id = $1 FOR UPDATE") + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to lock WAC parent {parent_job}: {e}")) + })?; + + if parked.is_some() { + let step_value = if success { + result.to_string() + } else { + let raw: Value = serde_json::from_str(result).unwrap_or(Value::Null); + wac_failure_record(&step_key, Some(&child_job.to_string()), &raw).to_string() + }; + let merged: Option = sqlx::query_scalar( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + workflow_as_code_status, + '{_checkpoint,completed_steps}', + COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) + || jsonb_build_object($2::text, $3::text::jsonb) + ) WHERE id = $1 + AND workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids'->>$2 = $4 + AND NOT COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps' ? $2, false) + RETURNING 1", + ) + .bind(parent_job) + .bind(&step_key) + .bind(&step_value) + .bind(child_job.to_string()) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to add WAC completed step: {e}")) + })?; + + if merged.is_some() { + // `suspend_until` stays set: the suspended pull query is what takes a + // parked parent back, and it selects on `suspend_until IS NOT NULL`. + let suspend: Option = sqlx::query_scalar( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 RETURNING suspend", + ) + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to unsuspend WAC parent: {e}")))?; + parent_ready = suspend == Some(0); + if parent_ready { + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' WHERE id = $1", + ) + .bind(parent_job) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to clear WAC pending steps: {e}")) + })?; + } + } + tracing::info!( + parent_job = %parent_job, + child_job = %child_job, + step_key = %step_key, + success, + recorded = merged.is_some(), + parent_ready, + "WAC v2 child job completed" + ); + } + } + + // The child's entry in the parent's timeline, keyed by child id. The parent may + // already be completed (cancelled with its children still running), in which + // case the entry lives on its completed row instead. Errors propagate: a failed + // statement has already aborted the transaction, so there is nothing to continue with. + let stamped: Option = sqlx::query_scalar( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + jsonb_set( + workflow_as_code_status, + ARRAY[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + ARRAY[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) WHERE id = $3 AND workflow_as_code_status IS NOT NULL RETURNING 1", + ) + .bind(child_job.to_string()) + .bind(duration_ms) + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not update parent job `duration_ms` in workflow as code status: {e}" + )) + })?; + if stamped.is_none() { + sqlx::query( + "UPDATE v2_job_completed SET workflow_as_code_status = jsonb_set( + jsonb_set( + workflow_as_code_status, + ARRAY[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + ARRAY[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) WHERE id = $3 AND workflow_as_code_status IS NOT NULL", + ) + .bind(child_job.to_string()) + .bind(duration_ms) + .bind(parent_job) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not update completed parent job `duration_ms` in workflow as code status: {e}" + )) + })?; + } + + Ok(parent_ready) +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 84284759c6..c742a3efcb 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -392,6 +392,7 @@ lazy_static::lazy_static! { pip_local_dependencies: Default::default(), env_vars: Default::default(), native_mode: false, + object_store_cache_config: Default::default(), }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -1081,6 +1082,83 @@ pub struct SqlAnnotations { pub raw_output: bool, } +impl SqlAnnotations { + /// The data table role a query declares as `-- role `, if any. Only meaningful against a + /// `datatable://` database that is under roles; absent means the data table's default role. + /// + /// Hand-written rather than derived because the value matters, not just the presence, and + /// because the executor needs it before it knows the connection is a data table at all. Like + /// every annotation it lives in the leading comment block. + /// + /// A leading comment whose first word is `role` is an annotation *attempt*, and a malformed + /// one is an error. The alternative — ignoring what does not parse — resolves the query to the + /// data table's default role instead, so a typo silently runs it under a login the author did + /// not choose, which is the opposite of what naming a role is for. Only callers that already + /// know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its + /// comments. + pub fn datatable_role(code: &str) -> error::Result> { + for line in code.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + if !line.starts_with("--") { + break; + } + // The keyword may be followed by whitespace, `:` or `=` — `role x`, `role: x`, + // `role=x`, `Role = x` all open an attempt, while `rolexyz` does not. Each accepted + // separator is one spelling that would otherwise take the `continue` below and run the + // query as the data table's default role, which is the silence this exists to remove. + let body = line[2..].trim_start(); + let Some(after) = body + .get(..4) + .filter(|kw| kw.eq_ignore_ascii_case("role")) + .map(|_| &body[4..]) + else { + continue; + }; + if !after.is_empty() + && !after.starts_with(char::is_whitespace) + && !after.starts_with([':', '=']) + { + continue; + } + + // Past this point the line is an attempt to name a role, so a malformed one is an + // error rather than a miss. Falling through would run the query as the data table's + // default role — quietly, and under a login the author did not choose. + let after = after.trim_start(); + let after = after.strip_prefix([':', '=']).unwrap_or(after); + let mut tokens = after.split_whitespace(); + let role = tokens + .next() + .map(|role| role.strip_suffix(';').unwrap_or(role)); + let rest = tokens.next(); + match (role, rest) { + (Some(role), None) + if !role.is_empty() + && role.len() <= 63 + && role + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') => + { + return Ok(Some(role.to_string())); + } + _ => { + return Err(error::Error::BadRequest(format!( + "Malformed data table role annotation: `{line}`. Write it as \ + `-- role ` on a line of its own, where is letters, digits, \ + '_' or '-'. A comment in the leading block that starts with the word \ + 'role' is read as this annotation; move it below the first statement if \ + it is prose." + ))); + } + } + } + Ok(None) + } +} + #[annotations("#")] pub struct BashAnnotations { pub docker: bool, @@ -2343,6 +2421,7 @@ pub async fn load_worker_config( .or_else(|| load_additional_python_paths_from_env()), env_vars: resolved_env_vars, native_mode, + object_store_cache_config: config.object_store_cache_config, }) } @@ -2432,6 +2511,7 @@ pub struct WorkerConfigOpt { pub env_vars_static: Option>, pub env_vars_allowlist: Option>, pub native_mode: Option, + pub object_store_cache_config: Option, } impl Default for WorkerConfigOpt { @@ -2450,6 +2530,7 @@ impl Default for WorkerConfigOpt { env_vars_static: Default::default(), env_vars_allowlist: Default::default(), native_mode: Default::default(), + object_store_cache_config: Default::default(), } } } @@ -2468,12 +2549,18 @@ pub struct WorkerConfig { pub pip_local_dependencies: Option>, pub env_vars: HashMap, pub native_mode: bool, + /// Object store this group's dependency cache uses instead of the instance one, as stored + /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate + /// that parses it, and comparing the raw value is what tells a reload the store changed. + pub object_store_cache_config: Option, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode) + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, + // holds bucket credentials + self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None")) } } @@ -2502,6 +2589,42 @@ pub fn split_python_requirements>(requirements: T) -> Vec .collect() } +/// Byte offset of the comment marker, per pip's rule: a `#` at line start or preceded by +/// whitespace. A `#` elsewhere belongs to the requirement (`pkg @ https://h/p.whl#sha256=…`). +fn requirement_comment_start(line: &str) -> Option { + line.char_indices() + .find(|(i, c)| *c == '#' && (*i == 0 || line[..*i].ends_with(char::is_whitespace))) + .map(|(i, _)| i) +} + +/// The installable requirement carried by one lockfile line, or `None` for a comment, a +/// `-r`/`-e`/`--flag` directive, or a blank. +/// +/// Windmill installs a lockfile one entry at a time as a `uv pip install` argument, so +/// requirements-file syntax a file-level parser would absorb is an unparseable package name +/// here and has to be stripped first. +pub fn requirement_from_lockfile_line(line: &str) -> Option<&str> { + let requirement = match requirement_comment_start(line) { + Some(i) => &line[..i], + None => line, + } + .trim() + // Continuations are stripped, not joined: right for `--generate-hashes` locks, whose + // continued lines are `--hash=` flags this function drops, but a lock continuing onto a + // marker or extra would lose it. + .trim_end_matches('\\') + .trim_end(); + + (!requirement.is_empty() && !requirement.starts_with('-')).then_some(requirement) +} + +/// Whether a lockfile line continues onto the next one. The continued lines reach the +/// installer as entries of their own rather than being joined, so a caller that cares what +/// they carried — `--hash=` pins, for a `--generate-hashes` lock — has to say so itself. +pub fn lockfile_line_has_continuation(line: &str) -> bool { + line.trim_end().ends_with('\\') +} + #[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] #[repr(u32)] pub enum PyVAlias { @@ -2607,6 +2730,56 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn datatable_role_is_read_from_the_leading_comment_block() { + let role = |code| SqlAnnotations::datatable_role(code); + assert_eq!( + role("-- role analytics\nSELECT 1").unwrap(), + Some("analytics".to_string()) + ); + // Blank lines and other annotations before it are fine. + assert_eq!( + role("\n-- prepare\n-- role read_only\nSELECT 1").unwrap(), + Some("read_only".to_string()) + ); + // Past the first statement it is an ordinary comment, not an annotation. + assert_eq!(role("SELECT 1;\n-- role analytics").unwrap(), None); + assert_eq!(role("SELECT 1").unwrap(), None); + + // Unambiguous intent is honoured: the keyword matches case-insensitively, a trailing + // semicolon is a habit carried over from SQL rather than a different role, and the colon + // spelling is the one most likely to be typed. + for accepted in [ + "-- Role operator\nSELECT 1", + "-- role operator;\nSELECT 1", + "-- role: operator\nSELECT 1", + "-- role:operator\nSELECT 1", + "-- role=operator\nSELECT 1", + "-- Role = operator\nSELECT 1", + ] { + assert_eq!( + role(accepted).unwrap(), + Some("operator".to_string()), + "not honoured: {accepted}" + ); + } + + // Anything else opening with the word is refused rather than resolved to the default role: + // the whole point of naming one is to not run as something else. + for near_miss in [ + "-- role operator -- why\nSELECT 1", + "-- role an;alytics\nSELECT 1", + "-- role\nSELECT 1", + "-- role:\nSELECT 1", + "-- role based access is handled below\nSELECT 1", + ] { + assert!(role(near_miss).is_err(), "silently ignored: {near_miss}"); + } + + // A word that merely starts with the keyword is not an attempt. + assert_eq!(role("-- rolebased notes\nSELECT 1").unwrap(), None); + } + fn matcher(id: &str) -> WorkspaceMatcher { WorkspaceMatcher { id: id.to_string(), include_forks: false } } @@ -2620,6 +2793,52 @@ mod tests { ids.iter().map(|s| s.to_string()).collect() } + /// Fixtures are verbatim `uv pip compile` output (uv 0.11.28): split and inline + /// annotation styles, and `--generate-hashes`. + #[test] + fn test_requirement_from_lockfile_line() { + assert_eq!(requirement_from_lockfile_line(" # via httpx"), None); + assert_eq!(requirement_from_lockfile_line(" # via"), None); + assert_eq!(requirement_from_lockfile_line(" # anyio"), None); + assert_eq!( + requirement_from_lockfile_line(" # via -r .tmp/requirements.in"), + None + ); + assert_eq!( + requirement_from_lockfile_line("anyio==4.15.1 \\"), + Some("anyio==4.15.1") + ); + assert_eq!( + requirement_from_lockfile_line( + " --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7 \\" + ), + None + ); + assert_eq!(requirement_from_lockfile_line("# py: 3.11"), None); + assert_eq!(requirement_from_lockfile_line("-r other.txt"), None); + assert_eq!( + requirement_from_lockfile_line("--index-url https://x"), + None + ); + assert_eq!(requirement_from_lockfile_line(" "), None); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0"), + Some("httpx==0.27.0") + ); + assert_eq!( + requirement_from_lockfile_line("httpx==0.27.0 # via -r requirements.in"), + Some("httpx==0.27.0") + ); + // A `#` not preceded by whitespace is part of the requirement, not a comment. + assert_eq!( + requirement_from_lockfile_line("wmill @ https://h/wmill.whl#sha256=abc"), + Some("wmill @ https://h/wmill.whl#sha256=abc") + ); + + assert!(lockfile_line_has_continuation("anyio==4.15.1 \\")); + assert!(!lockfile_line_has_continuation("anyio==4.15.1")); + } + #[test] fn test_parse_job_oom_score_adj() { assert_eq!(parse_job_oom_score_adj(Some("300")), 300); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index ae3e6546cd..6f82e3a261 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use strum::AsRefStr; use crate::{ + datatable_roles::{ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER}, error::{self, to_anyhow, Error, Result}, get_database_url, secret_backend::{get_secret_value, is_external_stored_value}, @@ -71,6 +72,7 @@ bitflags::bitflags! { const RESTRICT_DEPLOY_TO_DEPLOYERS = 1 << 2; const RESTRICT_ANONYMOUS_APP_DEPLOYMENT = 1 << 3; const RESTRICT_PUBLIC_RUN_SHARING = 1 << 4; + const RESTRICT_GUEST_APP_DEPLOYMENT = 1 << 5; } } @@ -83,6 +85,7 @@ pub enum ProtectionRuleKind { RestrictDeployToDeployers, RestrictAnonymousAppDeployment, RestrictPublicRunSharing, + RestrictGuestAppDeployment, } impl ProtectionRuleKind { @@ -103,6 +106,9 @@ impl ProtectionRuleKind { ProtectionRuleKind::RestrictPublicRunSharing => { ProtectionRules::RESTRICT_PUBLIC_RUN_SHARING } + ProtectionRuleKind::RestrictGuestAppDeployment => { + ProtectionRules::RESTRICT_GUEST_APP_DEPLOYMENT + } } } @@ -121,6 +127,9 @@ impl ProtectionRuleKind { ProtectionRuleKind::RestrictPublicRunSharing => { "Sharing a run publicly (readable without login) is restricted in this workspace" } + ProtectionRuleKind::RestrictGuestAppDeployment => { + "Opening an app to guests (anyone who can sign in) is restricted in this workspace" + } } } } @@ -175,7 +184,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -183,7 +192,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28910/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28970/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -337,13 +346,14 @@ pub struct GitRepositorySettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub auto_pull: Option, /// Open a PR when a deploy pushes a `wm_deploy/**` branch of this promotion - /// repo (app-backed only; runs from the deploy callback so it works without + /// repo (needs a credential the server holds — a GitHub App installation or + /// a checked GitLab token; runs from the deploy callback so it works without /// inbound webhooks). Off by default so upgrades don't change behavior. #[serde(default, skip_serializing_if = "is_false")] pub promotion_open_prs: bool, /// Parent-level: open a PR when a fork of this workspace deploys to its - /// `wm-fork/**` branch (app-backed only; the fork's deploy callback reads - /// this from the parent). Off by default. + /// `wm-fork/**` branch (needs a credential the server holds; the fork's + /// deploy callback reads this from the parent). Off by default. #[serde(default, skip_serializing_if = "is_false")] pub fork_open_prs: bool, /// Server-owned: the last failure opening a PR for a deploy branch of this @@ -352,6 +362,10 @@ pub struct GitRepositorySettings { /// successful PR; never accepted from clients. #[serde(default, skip_serializing_if = "Option::is_none")] pub open_pr_error: Option, + /// Server-owned: what the repo's credential says about its own expiry and + /// scopes. Written by the credential check, never accepted from clients. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential: Option, } impl GitRepositorySettings { @@ -397,6 +411,44 @@ pub enum AutoPullMode { Polling, } +/// Host whose credential lifecycle Windmill can manage from the repo URL. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum GitCredentialProvider { + Gitlab, +} + +/// What the repo's own credential says about itself, refreshed by asking the +/// host. Server-owned: written by the credential check, never accepted from a +/// client. +/// +/// Absent means the check has not run or the repo carries no credential we can +/// introspect (a GitHub App repo mints tokens per call and has nothing to expire). +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GitCredentialStatus { + pub provider: GitCredentialProvider, + /// Changes on every rotation, so it identifies the current token, not the + /// credential's whole history. + #[serde(skip_serializing_if = "Option::is_none")] + pub token_id: Option, + /// `None` is a non-expiring token, which only self-managed GitLab can issue + /// (and only for a service account). It means no warning and no rotation. + #[serde(skip_serializing_if = "Option::is_none")] + pub expires_at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scopes: Vec, + /// Whether *this workspace* renews the credential. That needs a scope which + /// permits it (`api` or `self_rotate`) and a credential this workspace holds: + /// a token carried in the repository URL is the operator's to manage, and one + /// resolved from an ancestor is the ancestor's, so neither is renewed here. + pub rotatable: bool, + /// Unix timestamp (seconds) of the last check. + pub checked_at: i64, + /// Why the last check or rotation failed, cleared by the next success. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + /// Outcome of the most recent auto-pull attempt, surfaced in the UI. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AutoPullStatus { @@ -463,6 +515,11 @@ pub struct AutoPullSettings { pub last_synced_sha: std::collections::HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_pull_status: Option, + /// Email of the admin this repository's automatic pulls (its own and its forks') + /// apply changes as: whoever last saved the settings with auto pull on. Stamped + /// server-side, never taken from the client. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled_by: Option, } // Manual Debug so the HMAC `webhook_secret` (even encrypted) never lands in logs. @@ -473,6 +530,7 @@ impl std::fmt::Debug for AutoPullSettings { .field("mode", &self.mode) .field("poll_interval_s", &self.poll_interval_s) .field("sync_forks", &self.sync_forks) + .field("enabled_by", &self.enabled_by) .field("webhook_id", &self.webhook_id) .field( "webhook_secret", @@ -508,9 +566,10 @@ impl AutoPullSettings { /// Whether a freshly observed `(git_ref, head_sha)` warrants enqueuing a pull. /// /// A trigger (poll or webhook) is only a hint: we pull when auto-pull is - /// enabled and the observed head differs from the last sha we synced for - /// that ref. Re-observing the same head (e.g. a redundant poll, or the - /// commit our own deploy callback just pushed back) is a no-op. + /// enabled and the observed head differs from the last sha we pulled for + /// that ref. Re-observing the same head (a redundant poll) is a no-op. A + /// commit our own deploy pushed is not: pushes never write here, so the pull + /// it triggers picks up anything pushed under it. pub fn should_pull(&self, git_ref: &str, head_sha: &str) -> bool { self.enabled && self.last_synced_sha.get(git_ref).map(String::as_str) != Some(head_sha) } @@ -759,15 +818,241 @@ pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result { Ok(count) } -/// Approximate paid seats of a workspace as `ceil(developers + operators/2)`, excluding disabled and -/// service-account members. Reuses billing's author/operator weighting, but counts provisioned -/// members rather than the active-user population billing meters, so it only ever loosens the fork -/// cap (never blocks a paid seat) — good enough for a soft guardrail. +/// The billable members of a workspace and the seats they add up to. +#[derive(Clone, Debug, Serialize)] +pub struct BillableSeats { + pub developers: i64, + pub operators: i64, + pub seats: i64, +} + +/// Guests are free up to `FREE_GUESTS_PER_WINDOW` distinct emails over the trailing +/// `GUEST_WINDOW_DAYS`. Past that, an Enterprise plan meters them, `GUESTS_PER_SEAT` +/// guests to one seat, while every other plan and build stops admitting new emails. +pub const GUEST_WINDOW_DAYS: i32 = 30; +pub const FREE_GUESTS_PER_WINDOW: i64 = 100; +pub const GUESTS_PER_SEAT: i64 = 4; + +/// Whether guests past the allowance are metered (Enterprise plan) rather than refused. +/// A build without `enterprise` has no plan and is capped, like a Pro key. +pub async fn guests_are_metered() -> bool { + #[cfg(feature = "enterprise")] + { + matches!( + crate::ee_oss::get_license_plan().await, + crate::ee_oss::LicensePlan::Enterprise + ) + } + #[cfg(not(feature = "enterprise"))] + { + false + } +} + +/// Seats the guests past the free allowance consume: `ceil(billable / GUESTS_PER_SEAT)`. +pub fn guest_seats(distinct_guests: i64) -> i64 { + let billable = (distinct_guests - FREE_GUESTS_PER_WINDOW).max(0); + (billable + GUESTS_PER_SEAT - 1) / GUESTS_PER_SEAT +} + +/// Distinct guest emails over the trailing window, today included. +pub async fn guest_count_in_window<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, +) -> Result { + sqlx::query_scalar( + "SELECT COUNT(DISTINCT email) FROM guest_activity WHERE day > CURRENT_DATE - $1", + ) + .bind(GUEST_WINDOW_DAYS) + .fetch_one(executor) + .await + .map_err(|e| Error::internal_err(format!("counting guests: {e:#}"))) +} + +/// The instance's standing against the guest allowance, as every surface reports it. +#[derive(Clone, Debug, Serialize)] +pub struct GuestUsage { + /// Whether this deployment can admit guests at all ([`instance_supports_guests`]). + /// Off, every other field is moot and no switch below can turn guests on. + pub available: bool, + /// The superadmin switch (`GUEST_ACCESS_DISABLED_SETTING`), which every workspace + /// switch sits under. Reported as stored, so a superadmin sees what they set even + /// where `available` overrules it. + pub instance_enabled: bool, + /// Distinct guest emails over the trailing `window_days`. + pub guest_count: i64, + pub window_days: i32, + pub free_allowance: i64, + /// Enterprise plan: guests past the allowance take `guest_seats`. Otherwise no new + /// email is admitted past it. + pub metered: bool, + pub billable_guests: i64, + pub guest_seats: i64, +} + +/// What a caller is told when it asks for guests on a deployment that cannot have them. +pub const GUESTS_UNAVAILABLE_MESSAGE: &str = + "Guest access is not available on Windmill Cloud. It requires a self-hosted instance \ + or a dedicated Windmill Cloud deployment."; + +/// Whether guests can exist on this deployment at all. They cannot on the shared cloud: +/// a guest is an identity Windmill itself never vouched for, admitted on the say-so of +/// whoever runs the instance, which is not a call a multi-tenant deployment can make for +/// its tenants. Folded into every guest gate below, so a workspace switch or an app +/// policy left saying `guest` is inert rather than honored. +pub fn instance_supports_guests() -> bool { + !*crate::worker::CLOUD_HOSTED +} + +/// [`instance_supports_guests`] as an error, for the writes that would otherwise store a +/// setting that can never take effect. +pub fn require_guest_support() -> Result<()> { + if instance_supports_guests() { + Ok(()) + } else { + Err(Error::BadRequest(GUESTS_UNAVAILABLE_MESSAGE.to_string())) + } +} + +/// SQL for the superadmin switch alone, absent meaning on. The setting is read as text +/// before the cast so `true` and `"true"` both count. +fn instance_switch_sql() -> String { + format!( + "NOT COALESCE((SELECT (value #>> '{{}}')::boolean FROM global_settings \ + WHERE name = '{}'), false)", + crate::global_settings::GUEST_ACCESS_DISABLED_SETTING + ) +} + +/// SQL for "the instance admits guests": the superadmin switch, under +/// [`instance_supports_guests`]. +fn instance_admits_guests_sql() -> String { + if !instance_supports_guests() { + return "false".to_string(); + } + instance_switch_sql() +} + +pub async fn guest_usage(db: &crate::DB) -> Result { + let instance_switch = instance_switch_sql(); + let instance_enabled: bool = sqlx::query_scalar(&format!("SELECT {instance_switch}")) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("reading the instance guest switch: {e:#}")))?; + let guest_count = guest_count_in_window(db).await?; + let metered = guests_are_metered().await; + let billable_guests = if metered { + (guest_count - FREE_GUESTS_PER_WINDOW).max(0) + } else { + 0 + }; + Ok(GuestUsage { + available: instance_supports_guests(), + instance_enabled, + guest_count, + window_days: GUEST_WINDOW_DAYS, + free_allowance: FREE_GUESTS_PER_WINDOW, + metered, + billable_guests, + guest_seats: if metered { guest_seats(guest_count) } else { 0 }, + }) +} + +/// Whether `email` may be admitted as a guest right now. Checked once, where a session +/// is minted: a returning guest (already in the window) is always let back in, so the +/// cap only ever refuses a stranger, and a metered instance refuses nobody. +/// +/// Must run inside the transaction that then records the guest in `guest_activity`: +/// it takes a transaction-scoped lock so concurrent strangers count each other, and the +/// lock is what keeps the cap exact rather than approximate. +pub async fn guest_admission(conn: &mut sqlx::PgConnection, email: &str) -> Result<()> { + if guests_are_metered().await { + return Ok(()); + } + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('guest_allowance'))") + .execute(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("locking the guest allowance: {e:#}")))?; + let (in_window, count): (bool, i64) = sqlx::query_as( + "SELECT + EXISTS(SELECT 1 FROM guest_activity WHERE email = $1 AND day > CURRENT_DATE - $2), + (SELECT COUNT(DISTINCT email) FROM guest_activity WHERE day > CURRENT_DATE - $2)", + ) + .bind(email) + .bind(GUEST_WINDOW_DAYS) + .fetch_one(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("checking the guest allowance: {e:#}")))?; + if in_window || count < FREE_GUESTS_PER_WINDOW { + return Ok(()); + } + Err(Error::PermissionDenied(format!( + "This instance has reached its limit of {FREE_GUESTS_PER_WINDOW} guests over \ + {GUEST_WINDOW_DAYS} days. Guest sign-in beyond that needs an Enterprise license." + ))) +} + +/// Whether a guest session for `email` in `w_id` still stands: the instance and the +/// workspace admit guests, and the email still has no account. Read at the auth door on +/// every guest request, so turning either switch off, or an account provisioned after +/// the mint (or racing it), ends the session on its next request. +pub async fn guest_session_stands(db: &crate::DB, w_id: &str, email: &str) -> Result { + let instance_admits = instance_admits_guests_sql(); + let stands: Option = sqlx::query_scalar(&format!( + "SELECT guest_access_enabled + AND {instance_admits} + AND NOT EXISTS(SELECT 1 FROM password WHERE email = $2) + AND NOT EXISTS(SELECT 1 FROM usr WHERE email = $2) + FROM workspace_settings WHERE workspace_id = $1" + )) + .bind(w_id) + .bind(email) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("checking the guest session of {email}: {e:#}")))?; + Ok(stands.unwrap_or(false)) +} + +/// Every switch at once: the instance's, the workspace's, and `app_path` being in +/// `guest` execution mode. The single answer to "may a guest session be minted for this +/// app", used by the mint itself and by the sign-in branch that decides whether to call +/// it. A missing app or a policy with no stated mode reads as "no". The allowance is +/// `guest_admission`. +pub async fn guest_app_admits<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, + w_id: &str, + app_path: &str, +) -> Result { + // The mint refuses a path it cannot scope, so discovery must not advertise one. + if !crate::auth::is_scope_literal_path(app_path) { + return Ok(false); + } + let instance_admits = instance_admits_guests_sql(); + let admits: Option = sqlx::query_scalar(&format!( + "SELECT COALESCE(ws.guest_access_enabled AND app.policy->>'execution_mode' = 'guest', false) + AND {instance_admits} + FROM app JOIN workspace_settings ws ON ws.workspace_id = app.workspace_id + WHERE app.workspace_id = $1 AND app.path = $2" + )) + .bind(w_id) + .bind(app_path) + .fetch_optional(executor) + .await + .map_err(|e| { + Error::internal_err(format!("checking guest access to {w_id}/{app_path}: {e:#}")) + })?; + Ok(admits.unwrap_or(false)) +} + +/// Billable members of `w_id` and the seats they cost, as `ceil(developers + operators/2)`. Service +/// accounts cannot log in and do not take a seat; a disabled member is not billed either. +/// +/// The workspace is invoiced by a job outside this codebase that counts the same rows with its own +/// SQL. The two must be changed together: this rule disagreeing with that one is what bills a +/// workspace for seats the product never credits it for. /// /// Unauthenticated metering helper: reads member counts for any `w_id`, so callers must already be /// authorized for that workspace (or run in trusted server-side code). -#[cfg(feature = "cloud")] -pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { +pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result { let row = sqlx::query!( r#"SELECT COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!", @@ -777,8 +1062,18 @@ pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { ) .fetch_one(db) .await - .map_err(|e| Error::internal_err(format!("counting paid seats of {w_id}: {e:#}")))?; - Ok(((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64) + .map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?; + Ok(BillableSeats { + developers: row.developers, + operators: row.operators, + seats: ((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64, + }) +} + +/// Seats only, for the fork cap. See [`billable_seats`]. +#[cfg(feature = "cloud")] +pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { + Ok(billable_seats(db, w_id).await?.seats) } #[cfg(feature = "cloud")] @@ -998,9 +1293,23 @@ impl Default for DataTableForkBehavior { } } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct DataTable { - pub database: DataTableDatabase, + /// Set on a *terminal* entry — one that owns its database. Mutually exclusive with + /// [`DataTable::reference`]; [`validate_datatable_shape`] is the one place that enforces it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database: Option, + /// Set on a *pointer* entry — one that names another workspace's entry and owns nothing. + /// A keep-original fork gets one of these instead of a copy of the parent's entry, so there is + /// nothing local for a fork admin to widen. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reference: Option, + /// Set on a *clone* — a terminal entry whose database was copied from the entry this names. The + /// copy holds that entry's rows, so who may connect as which role stays that entry's decision: + /// the clone carries no `permissions` of its own and is governed like a pointer, while + /// connecting to its own database. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub governed_by: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub forked_from: Option, /// Whether the SQL-migrations feature is opted in for this data table. @@ -1008,22 +1317,92 @@ pub struct DataTable { /// when migrations already exist (see `datatable_migrations_enabled`). #[serde(default, skip_serializing_if = "Option::is_none")] pub migrations_enabled: Option, + /// Who may connect as which role. Absent = unpermissioned: every caller connects as `admin`, + /// which is how data tables behaved before roles existed. Only meaningful on a terminal entry; + /// a pointer is governed by what it points at. + /// + /// Never leaves the instance: stripped from the workspace export and ignored on import, since + /// tenants are workspace-scoped names and syncing them would make repo write access a second + /// door onto the access decision. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub permissions: Option, } -#[derive(Deserialize, Serialize, Debug)] +/// A pointer at another workspace's data table entry. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct DataTableReference { + pub workspace_id: String, + pub datatable: String, +} + +/// The access decision for one data table: which role a caller gets, and who may ask for each. +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct DataTablePermissions { + /// A role id from the instance catalog, or `admin`. Absent = `admin`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_role: Option, + /// Keyed by instance role id, plus the reserved `admin` key. A role absent from this map + /// cannot be used on this data table at all, whatever the instance catalog says. + #[serde(default)] + pub roles: std::collections::BTreeMap, +} + +impl DataTablePermissions { + pub fn default_role(&self) -> &str { + self.default_role.as_deref().unwrap_or(ADMIN_DATATABLE_ROLE) + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct DataTableRoleTenants { + /// `u/`, `g/`, `f/`, or `*` for every member. + #[serde(default)] + pub tenants: Vec, +} + +/// Every member of the governing workspace. +pub const DATATABLE_TENANT_WILDCARD: &str = "*"; + +/// How deep a chain of pointer entries may go before it is called a loop. Data tables are not +/// expected to chain at all — a fork points at its parent — so this only has to be generous +/// enough to survive a fork of a fork. +const DATATABLE_REFERENCE_MAX_DEPTH: usize = 20; + +/// Exactly one of `database` and `reference` must be set, and a clone's `governed_by` leaves no +/// `permissions` beside it. Called wherever an entry is persisted, so nothing downstream has to +/// handle an entry that is both or neither, or a clone with a decision of its own. +pub fn validate_datatable_shape(name: &str, dt: &DataTable) -> Result<()> { + if dt.governed_by.is_some() && (dt.database.is_none() || dt.permissions.is_some()) { + return Err(Error::BadRequest(format!( + "Data table '{name}' is a clone, which owns a database and takes its roles from the \ + data table it was cloned from" + ))); + } + match (&dt.database, &dt.reference) { + (Some(_), None) | (None, Some(_)) => Ok(()), + (Some(_), Some(_)) => Err(Error::BadRequest(format!( + "Data table '{name}' both owns a database and points at another one" + ))), + (None, None) => Err(Error::BadRequest(format!( + "Data table '{name}' names neither a database nor another data table" + ))), + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct DataTableForkedFrom { /// Schema snapshot at fork time #[serde(default, skip_serializing_if = "Option::is_none")] pub schema: Option, } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct DataTableDatabase { pub resource_type: DataTableCatalogResourceType, pub resource_path: String, } -#[derive(Deserialize, Serialize, Debug, PartialEq)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone, Copy)] #[serde(rename_all = "lowercase")] #[derive(AsRefStr)] #[strum(serialize_all = "lowercase")] @@ -1059,37 +1438,13 @@ fn datatable_not_found_error(name: &str, datatables: Option<&serde_json::Value>) )) } -pub async fn get_datatable_resource_from_db_unchecked( - db: &DB, - w_id: &str, - name: &str, -) -> Result { - get_datatable_resource_inner(db, w_id, name, false).await -} - -/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger -/// connections: custom-instance datatables resolve to -/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres -/// datatables resolve to the user's own resource unchanged; configuring it for -/// replication there is the user's responsibility. +/// Read one workspace's data table entry, without following a pointer. /// -/// Authorization: like its `_unchecked` sibling, returns resolved connection -/// credentials and performs no authorization — callers MUST have already authorized -/// access to the datatable (e.g. the trigger's own create-time check). -pub async fn get_datatable_replication_resource_from_db_unchecked( - db: &DB, - w_id: &str, - name: &str, -) -> Result { - get_datatable_resource_inner(db, w_id, name, true).await -} - -async fn get_datatable_resource_inner( - db: &DB, - w_id: &str, - name: &str, - replication: bool, -) -> Result { +/// Disclosure: this is the primitive [`resolve_governing_datatable`] calls on every path, so it is +/// deliberately open to anything that has to resolve a data table, including for a workspace the +/// caller does not belong to. What it returns is not: callers MUST NOT put `permissions` into a +/// response, an export or a log — it names the governing workspace's users, groups and folders. +pub async fn read_datatable_entry(db: &DB, w_id: &str, name: &str) -> Result { let datatables = sqlx::query_scalar!( r#" SELECT ws.datatable->'datatables' AS datatables @@ -1107,37 +1462,674 @@ async fn get_datatable_resource_inner( .and_then(|d| d.get(name)) .filter(|v| !v.is_null()) .ok_or_else(|| datatable_not_found_error(name, datatables.as_ref()))?; - let datatable = serde_json::from_value::(datatable.clone())?; + Ok(serde_json::from_value::(datatable.clone())?) +} - let db_resource = if datatable.database.resource_type == DataTableCatalogResourceType::Instance - { +/// The terminal entry a reference chain lands on, and what governs it: the workspace and name of +/// the entry that owns the database, the entry itself, and — for a clone — the entry its +/// `governed_by` chain lands on. A terminal entry that is not a clone resolves to itself and +/// governs itself. +/// +/// Every decision downstream is taken on this, never on the entry the caller named. Which database +/// to connect to comes from `workspace_id` / `name` / `datatable.database`; whose `permissions` +/// apply (already in `datatable.permissions`), whose members tenants are evaluated against and who +/// may administer it come from [`GoverningDatatable::governing_workspace_id`]. +/// +/// Authorization: resolving deliberately crosses into the governing workspace, so it answers for a +/// workspace the caller may not belong to and checks nothing itself. It is the input to the +/// checks, not one of them: callers MUST pass what it returns to +/// [`can_use_datatable_role_in_governing_workspace`] or [`ensure_datatable_admin_access`] before +/// acting on it, and MUST NOT return its `permissions` or workspace ids to a caller from +/// elsewhere without gating on the answer. +pub struct GoverningDatatable { + pub workspace_id: String, + pub name: String, + pub datatable: DataTable, + /// For a clone, the entry whose `permissions` govern it. `None` when the entry governs itself. + pub governor: Option, +} + +impl GoverningDatatable { + /// Backed by the Windmill instance's own Postgres, which is the only substrate data table + /// roles apply to. + pub fn is_instance(&self) -> bool { + self.datatable + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) + } + + /// The workspace whose admins administer the data table and whose members its tenants are. + pub fn governing_workspace_id(&self) -> &str { + self.governor + .as_ref() + .map_or(&self.workspace_id, |g| &g.workspace_id) + } +} + +pub async fn resolve_governing_datatable( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let mut workspace_id = w_id.to_string(); + let mut name = name.to_string(); + let mut hops = 0; + // The clone a `governed_by` chain started from: it keeps its database, and takes the + // `permissions` of wherever the chain lands. + let mut clone: Option<(String, String, DataTable)> = None; + for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH { + let datatable = read_datatable_entry(db, &workspace_id, &name) + .await + .map_err(|e| { + if hops == 0 { + e + } else { + // A pointer outlives the workspace it names: deleting one only nulls the fork + // lineage, it does not sweep the entries that pointed at it. Say which one is + // gone rather than reporting a data table this workspace never had. + if clone.is_some() { + Error::NotFound(format!( + "Data table '{name}' of workspace '{workspace_id}', which this clone \ + takes its roles from, no longer exists, so nobody is let into the copy." + )) + } else { + Error::NotFound(format!( + "Data table '{name}' of workspace '{workspace_id}' governs this one and \ + no longer exists. A superadmin can point this data table somewhere \ + else." + )) + } + } + })?; + hops += 1; + validate_datatable_shape(&name, &datatable)?; + let next = match (&datatable.reference, &datatable.governed_by) { + (Some(reference), _) => reference.clone(), + (None, Some(governed_by)) => { + let governed_by = governed_by.clone(); + if clone.is_none() { + clone = Some((workspace_id.clone(), name.clone(), datatable)); + } + governed_by + } + (None, None) => { + return Ok(match clone { + None => GoverningDatatable { workspace_id, name, datatable, governor: None }, + Some((clone_w_id, clone_name, mut clone_datatable)) => { + clone_datatable.permissions = datatable.permissions; + GoverningDatatable { + workspace_id: clone_w_id, + name: clone_name, + datatable: clone_datatable, + governor: Some(DataTableReference { workspace_id, datatable: name }), + } + } + }); + } + }; + workspace_id = next.workspace_id; + name = next.datatable; + } + Err(Error::BadRequest(format!( + "Data table '{name}' points at another data table through more than \ + {DATATABLE_REFERENCE_MAX_DEPTH} hops; the chain is likely a loop" + ))) +} + +/// Every entry of a workspace resolved as [`resolve_governing_datatable`] resolves one, in stored +/// order, reading the settings rows one pointer hop at a time rather than once per entry. An entry +/// that does not resolve — malformed, a dangling pointer, a loop — is left out. Same authorization +/// contract as the single resolution: it checks nothing. +pub async fn resolve_workspace_governing_datatables( + db: &DB, + w_id: &str, +) -> Result> { + type Entries = + std::collections::HashMap>; + async fn load(db: &DB, workspaces: &[String], entries: &mut Entries) -> Result> { + let rows: Vec<(String, String, serde_json::Value)> = sqlx::query_as( + "SELECT ws.workspace_id, dt.key, dt.value FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id = ANY($1)", + ) + .bind(workspaces) + .fetch_all(db) + .await?; + for ws in workspaces { + entries.entry(ws.clone()).or_default(); + } + let mut keys = Vec::with_capacity(rows.len()); + for (ws, key, value) in rows { + keys.push(key.clone()); + entries.entry(ws).or_default().insert(key, value); + } + Ok(keys) + } + + let mut entries = Entries::new(); + let listed = load(db, &[w_id.to_string()], &mut entries).await?; + // (index into `listed`, workspace, entry name, the clone the chain started from) still to be + // followed. A clone keeps its own database and takes the `permissions` of wherever its + // `governed_by` chain lands, as the single resolution does. + type Clone = Option<(String, String, DataTable)>; + let mut cursors: Vec<(usize, String, String, Clone)> = listed + .iter() + .enumerate() + .map(|(i, name)| (i, w_id.to_string(), name.clone(), None)) + .collect(); + let mut resolved: Vec<(usize, GoverningDatatable)> = vec![]; + + for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH { + let mut next = vec![]; + for (i, ws, name, clone) in cursors.drain(..) { + let Some(value) = entries + .get(&ws) + .and_then(|m| m.get(&name)) + .filter(|v| !v.is_null()) + else { + continue; + }; + let Ok(datatable) = serde_json::from_value::(value.clone()) else { + continue; + }; + if validate_datatable_shape(&name, &datatable).is_err() { + continue; + } + match (&datatable.reference, &datatable.governed_by) { + (Some(reference), _) => next.push(( + i, + reference.workspace_id.clone(), + reference.datatable.clone(), + clone, + )), + (None, Some(governed_by)) => { + let (governor_ws, governor_name) = + (governed_by.workspace_id.clone(), governed_by.datatable.clone()); + let clone = clone.or(Some((ws, name, datatable))); + next.push((i, governor_ws, governor_name, clone)); + } + (None, None) => resolved.push(( + i, + match clone { + None => GoverningDatatable { + workspace_id: ws, + name, + datatable, + governor: None, + }, + Some((clone_ws, clone_name, mut clone_datatable)) => { + clone_datatable.permissions = datatable.permissions; + GoverningDatatable { + workspace_id: clone_ws, + name: clone_name, + datatable: clone_datatable, + governor: Some(DataTableReference { + workspace_id: ws, + datatable: name, + }), + } + } + }, + )), + } + } + if next.is_empty() { + break; + } + let to_load: Vec = next + .iter() + .map(|(_, ws, _, _)| ws.clone()) + .filter(|ws| !entries.contains_key(ws)) + .collect::>() + .into_iter() + .collect(); + if !to_load.is_empty() { + load(db, &to_load, &mut entries).await?; + } + cursors = next; + } + + resolved.sort_by_key(|(i, _)| *i); + Ok(resolved + .into_iter() + .map(|(i, governing)| (listed[i].clone(), governing)) + .collect()) +} + +/// Build the `admin` connection for a governing entry: `custom_instance_user` for an instance +/// database, the user's own resource for a BYO-postgres one. +async fn resolve_datatable_connection_unchecked( + db: &DB, + governing: &GoverningDatatable, + replication: bool, +) -> Result { + let database = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database"); + if database.resource_type == DataTableCatalogResourceType::Instance { let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; - pg_creds.dbname = datatable.database.resource_path.clone(); + pg_creds.dbname = database.resource_path.clone(); if replication { pg_creds.user = Some("custom_instance_replication_user".to_string()); pg_creds.password = Some(get_custom_pg_instance_replication_password(&db).await?); } else { - pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.user = Some(CUSTOM_INSTANCE_USER.to_string()); pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); } serde_json::to_value(&pg_creds) - .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))? + .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e))) } else { // Name the data table too: the caller asked for one by name, and a bare // "resource f/x/y does not exist" leaves them to work out which one points at it. transform_json_unchecked( - &serde_json::Value::String(format!("$res:{}", datatable.database.resource_path)), - w_id, + &serde_json::Value::String(format!("$res:{}", database.resource_path)), + &governing.workspace_id, db, ) .await .map_err(|e| match e { - Error::NotFound(m) => Error::NotFound(format!("data table {name}: {m}")), + Error::NotFound(m) => Error::NotFound(format!("data table {}: {m}", governing.name)), e => e, - })? + }) + } +} + +/// Resolve a data table to connection credentials **without authorizing anything**: always the +/// `admin` connection. +/// +/// Authorization: callers MUST have authorized access already. Anything that acts for a user or a +/// job wants [`get_datatable_resource_from_db`] instead. +pub async fn get_datatable_resource_from_db_unchecked( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + resolve_datatable_connection_unchecked(db, &governing, false).await +} + +/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger +/// connections: custom-instance datatables resolve to +/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres +/// datatables resolve to the user's own resource unchanged; configuring it for +/// replication there is the user's responsibility. +/// +/// Authorization: a replication connection reads every row whatever the roles grant, so no role or +/// admin check makes it safe. Callers MUST refuse a data table under roles outright — the Postgres +/// trigger crate's `ensure_not_under_roles` — and turning roles on is refused while one streams. +pub async fn get_datatable_replication_resource_from_db_unchecked( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + resolve_datatable_connection_unchecked(db, &governing, true).await +} + +/// The identity a resolution is made for. `Unchecked` is for callers that authorized already; +/// everything else is checked against the governing entry's tenants. +pub enum DatatableAccess<'a> { + /// Reaches every role. For callers that already authorized, or that have no user at all. + Unchecked, + Authed(crate::db::AuthedRef<'a>), + /// A job's owner, without reading the job row — only fetched if the data table turns out to + /// be permissioned. + PermissionedAs { + permissioned_as: &'a str, + email: &'a str, + }, + /// A job identified by id; its owner is read from `v2_job`. For agent workers and anything + /// else that authenticates as infrastructure rather than as the job's user. + Job(uuid::Uuid), + /// No identity established. Unpermissioned data tables resolve as before; permissioned ones + /// are refused, so a caller predating this feature fails closed. + NoIdentity, +} + +/// Does one tenant list cover this identity? Admins of the governing workspace pass everything — +/// they can edit the tenant lists anyway, so refusing them would only be theatre. +pub fn can_use_datatable_role( + tenants: &DataTableRoleTenants, + authed: &crate::db::AuthedRef<'_>, +) -> bool { + crate::datatable_roles_oss::can_use_datatable_role(tenants, authed) +} + +/// Evaluate a tenant list **as a member of the governing workspace**, whoever is calling. +/// +/// A caller reaching a data table through a pointer is a member of some other workspace, and being +/// its admin means nothing here — that is the whole point of the pointer. They are looked up in +/// the governing workspace by email and evaluated there, or refused when they are not a member. +/// A `g/` or `f/` permissioned-as from a foreign workspace is refused outright: those names are +/// defined per workspace and mean nothing outside the one that defined them. +pub async fn can_use_datatable_role_in_governing_workspace( + db: &DB, + governing_w_id: &str, + w_id: &str, + tenants: &DataTableRoleTenants, + access: &DatatableAccess<'_>, +) -> Result { + crate::datatable_roles_oss::can_use_datatable_role_in_governing_workspace( + db, + governing_w_id, + w_id, + tenants, + access, + ) + .await +} + +/// Resolve a data table to connection credentials for one identity. +/// +/// This is the chokepoint: everything that opens a connection to a data table on someone's behalf +/// goes through it. `role` is the role name the caller asked for — the `-- role` annotation, the +/// `?role=` on a `datatable://` reference, or `None` for the data table's default. +/// +/// The resolved role **logs in as itself**. Never `SET ROLE`: a script could `RESET ROLE` its way +/// back to admin. +pub async fn get_datatable_resource_from_db( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + access: DatatableAccess<'_>, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + let db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?; + // Not under roles and asked for none, or for `admin` by name: the `admin` connection, as before + // roles existed, in every edition. Anything else is a role decision. Every migration names + // `admin` explicitly, so an edition without roles must not treat that as one. + if governing.datatable.permissions.is_none() && role.is_none_or(|r| r == ADMIN_DATATABLE_ROLE) { + return Ok(db_resource); + } + crate::datatable_roles_oss::resolve_datatable_role_connection( + db, + w_id, + name, + &governing, + db_resource, + role, + access, + ) + .await +} + +/// Would the chokepoint accept this identity connecting as this role? Answers without resolving +/// credentials, for callers that want to refuse early and say which thing was refused. +/// +/// Not the security boundary — [`get_datatable_resource_from_db`] re-checks when it actually opens +/// the connection. This is what turns "permission denied for table x" into a message naming the +/// migration and the role. +pub async fn ensure_can_use_datatable_role( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + access: &DatatableAccess<'_>, + context: &str, +) -> Result<()> { + crate::datatable_roles_oss::ensure_can_use_datatable_role(db, w_id, name, role, access, context) + .await +} + +/// Gate the operations that see the whole database whatever the roles grant: a migration that +/// declares no role, exports, and editing the permissions themselves. Not replication, which a +/// data table under roles refuses whoever asks (see `ensure_not_under_roles`). Passing +/// means the caller could have connected as `admin` anyway. +pub async fn ensure_datatable_admin_access( + db: &DB, + w_id: &str, + name: &str, + access: &DatatableAccess<'_>, +) -> Result<()> { + crate::datatable_roles_oss::ensure_datatable_admin_access(db, w_id, name, access).await +} + +/// Rewrite the `permissions` of every data table entry of one workspace, in the caller's +/// transaction. `change` reports whether it touched anything; the row is only written when +/// something did. +/// +/// Authorization: writes an access decision for any workspace named, with an arbitrary mutation, +/// and checks nothing. It exists for the cascades below — the transaction that frees or renames a +/// principal — so callers MUST be the operation that made the principal change, and MUST run in +/// its transaction. Anything editing a decision on purpose belongs in the permissions endpoint, +/// which is gated on the workspace that governs the data table. +/// +/// The tenant lists name principals of this workspace, so anything that frees or renames one has +/// to come through here in the same transaction that frees it — otherwise a `u/alice` reused by a +/// later account silently inherits her access. +pub async fn update_datatable_permissions_in_workspace( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + change: F, +) -> Result<()> +where + F: Fn(&mut DataTablePermissions) -> bool, +{ + let Some(mut settings) = sqlx::query_scalar!( + "SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + w_id + ) + .fetch_optional(&mut **tx) + .await? + .flatten() else { + return Ok(()); }; - Ok(db_resource) + let Some(datatables) = settings + .get_mut("datatables") + .and_then(|d| d.as_object_mut()) + else { + return Ok(()); + }; + + let mut touched = false; + for entry in datatables.values_mut() { + let Some(permissions) = entry.get("permissions").filter(|p| !p.is_null()) else { + continue; + }; + let Ok(mut permissions) = + serde_json::from_value::(permissions.clone()) + else { + continue; + }; + if change(&mut permissions) { + entry["permissions"] = serde_json::to_value(&permissions) + .map_err(|e| Error::internal_err(format!("serializing permissions: {e}")))?; + touched = true; + } + } + + if touched { + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + settings, + w_id + ) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +/// Drop a freed principal (`u/alice`, `g/analysts`, `f/finance`) from every tenant list of one +/// workspace. Same contract as [`update_datatable_permissions_in_workspace`]: for the transaction +/// that frees the principal, not for editing a decision. +pub async fn remove_datatable_tenant_in_workspace( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + tenant: &str, +) -> Result<()> { + update_datatable_permissions_in_workspace(tx, w_id, |permissions| { + let mut touched = false; + for role in permissions.roles.values_mut() { + let before = role.tenants.len(); + role.tenants.retain(|t| t != tenant); + touched |= role.tenants.len() != before; + } + touched + }) + .await +} + +/// Follow a renamed principal through every tenant list of one workspace. Same contract as +/// [`update_datatable_permissions_in_workspace`]. +pub async fn rename_datatable_tenant_in_workspace( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + old: &str, + new: &str, +) -> Result<()> { + update_datatable_permissions_in_workspace(tx, w_id, |permissions| { + let mut touched = false; + for role in permissions.roles.values_mut() { + let mut role_touched = false; + for tenant in role.tenants.iter_mut() { + if tenant == old { + *tenant = new.to_string(); + role_touched = true; + } + } + if role_touched { + // The rename can collide with a name already in the list, and the two need not be + // adjacent — `Vec::dedup` only collapses neighbours, so it would leave the pair. + let mut seen = std::collections::HashSet::new(); + role.tenants.retain(|t| seen.insert(t.clone())); + touched = true; + } + } + touched + }) + .await +} + +/// Strip a deleted instance role from every workspace that had tenanted it, so nothing is left +/// naming a role that no longer exists. +/// +/// Authorization: reaches every workspace on the instance. Callers MUST be the superadmin path +/// dropping the role from the cluster — it exists to follow that, not to edit tenants. +/// +/// Takes that path's transaction rather than opening its own: run afterwards, a failure part-way +/// leaves the catalog row already gone, so the retry answers `NotFound` while some workspaces +/// still name a role nothing can connect as. In the transaction, the cluster drop, the catalog row +/// and every tenant list commit together or not at all. A data table whose default role was the deleted one falls +/// back to `admin` — the one role that is always present. +pub async fn forget_datatable_role_everywhere( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + role_id: &str, +) -> Result<()> { + crate::datatable_roles_oss::forget_datatable_role_everywhere(tx, role_id).await +} + +/// Drop the `permissions` block from a `workspace_settings.datatable` value before it leaves the +/// server. +/// +/// Who may connect as which role is an access decision, not configuration, and its tenants name +/// principals of one workspace — `g/analysts` in dev is a different group from `g/analysts` in +/// prod. Shipping it would both mean nothing at the far end and turn a settings push into a way to +/// widen access, so the decision stays where it was made. [`DataTable`] deserializes fine without +/// it, and the settings-editing endpoint carries the stored block across untouched. +pub fn strip_datatable_permissions( + datatable: Option, +) -> Option { + let mut datatable = datatable?; + if let Some(entries) = datatable + .get_mut("datatables") + .and_then(|d| d.as_object_mut()) + { + for entry in entries.values_mut() { + if let Some(entry) = entry.as_object_mut() { + entry.remove("permissions"); + } + } + } + Some(datatable) +} + +/// As [`parse_datatable_ref`], except that an entry whose stored name itself contains `?` — which +/// names could before they were restricted — resolves by that exact name, without a role. It is +/// looked up first, so `sales?role=x` never reaches a different entry than the one stored so. +/// When `sales` is stored too, the reference means either one, and is refused rather than +/// resolved to whichever is looked up first. +/// +/// Authorization: checks nothing, and its answer reveals whether `w_id` stores that exact name. +/// Callers MUST already act for `w_id` — a job of it, or a caller authenticated into it — and +/// MUST still pass the name to [`get_datatable_resource_from_db`] or an admin-access check. +pub async fn parse_datatable_ref_for( + db: &DB, + w_id: &str, + reference: &str, +) -> Result<(String, Option)> { + if reference.contains('?') { + let role_target = parse_datatable_ref(reference) + .ok() + .and_then(|(name, role)| role.map(|_| name)); + let (exists, target_exists) = sqlx::query_as::<_, (Option, Option)>( + "SELECT (datatable->'datatables') ? $2, (datatable->'datatables') ? $3 + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .bind(reference) + .bind(role_target) + .fetch_optional(db) + .await? + .unwrap_or((None, None)); + if exists.unwrap_or(false) { + if let (Some(name), Some(true)) = (role_target, target_exists) { + return Err(Error::BadRequest(format!( + "Data table reference '{reference}' names both the data table '{reference}' \ + and a role on the data table '{name}'. Rename '{reference}' to use either." + ))); + } + return Ok((reference.to_string(), None)); + } + } + let (name, role) = parse_datatable_ref(reference)?; + Ok((name.to_string(), role.map(str::to_string))) +} + +/// Split a `datatable://` reference into its name and the role its query string names. +/// +/// A query string that does not parse is an error rather than an absent role. Falling back would +/// resolve the reference to the data table's default role, so `?Role=analytics` or a mistyped +/// `?role=` would quietly connect as something the caller did not ask for — the same trap as a +/// malformed `-- role` annotation, and `role` is the only parameter a reference takes. +pub fn parse_datatable_ref(reference: &str) -> Result<(&str, Option<&str>)> { + let (name, query) = reference.split_once('?').unwrap_or((reference, "")); + let mut role = None; + for param in query.split('&').filter(|p| !p.is_empty()) { + let (key, value) = param.split_once('=').unwrap_or((param, "")); + if !key.eq_ignore_ascii_case("role") { + return Err(Error::BadRequest(format!( + "Data table reference '{name}' carries an unknown parameter '{key}'. \ + The only one it takes is `?role=`." + ))); + } + if role.is_some() { + return Err(Error::BadRequest(format!( + "Data table reference '{name}' names a role more than once." + ))); + } + if value.is_empty() || !is_datatable_role_name(value) { + return Err(Error::BadRequest(format!( + "Data table reference '{name}' has a malformed role '{value}'. Write it as \ + `?role=`, where is letters, digits, '_' or '-'." + ))); + } + role = Some(value); + } + Ok((name, role)) +} + +fn is_datatable_role_name(role: &str) -> bool { + !role.is_empty() + && role.len() <= 63 + && role + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') } #[derive(Deserialize, Serialize, Debug)] @@ -1494,6 +2486,48 @@ pub async fn workspace_with_fork_ancestors(db: &crate::DB, w_id: &str) -> Result Ok(chain) } +/// The nearest fork ancestor of `w_id` holding a row at `path` in `table`, or `None` when no +/// ancestor does (or `w_id` is not a fork). Fork creation clones trigger and schedule rows down +/// the whole chain, so a row shares its upstream identifier (Kafka group, PG slot, cron) with +/// every ancestor that still has one, not just the direct parent, which may have deleted its +/// copy since. +/// +/// Runs on the caller's connection so it sees the caller's transaction, and uncached: the +/// answer depends on the target table, not only on lineage. +/// +/// `table` is interpolated into SQL, hence `'static`: a trigger's `TABLE_NAME` or a literal, +/// never caller input. Reads lineage for any `w_id` with no authorization check, like +/// [`fork_ancestor_chain`], so the caller must already be authorized for `w_id`. +pub async fn nearest_fork_ancestor_having( + conn: &mut sqlx::PgConnection, + table: &'static str, + w_id: &str, + path: &str, +) -> Result> { + sqlx::query_scalar(&format!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT chain.id FROM chain + JOIN {table} t ON t.workspace_id = chain.id AND t.path = $2 + WHERE chain.depth > 0 + ORDER BY chain.depth LIMIT 1 + "# + )) + .bind(w_id) + .bind(path) + .fetch_optional(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("resolving fork ancestors of {w_id}: {e:#}"))) +} + lazy_static::lazy_static! { /// workspace id -> (root workspace id, expiry ts). Read once per job start, so correctness /// rests on the invalidation rather than on the TTL: every mutation that can change the answer @@ -2306,6 +3340,131 @@ async fn transform_json_unchecked( mod tests { use super::*; + fn tenants(list: &[&str]) -> DataTableRoleTenants { + DataTableRoleTenants { tenants: list.iter().map(|t| t.to_string()).collect() } + } + + #[cfg(all(feature = "private", feature = "enterprise"))] + #[test] + fn a_tenant_list_covers_users_groups_folders_and_the_wildcard() { + let groups = vec!["analysts".to_string()]; + let folders = vec![("finance".to_string(), true, false)]; + let scopes = None; + let token_prefix = None; + let is_admin = false; + let is_operator = false; + let authed = crate::db::AuthedRef { + email: "alice@windmill.dev", + username: "alice", + is_admin: &is_admin, + is_operator: &is_operator, + groups: &groups, + folders: &folders, + scopes: &scopes, + token_prefix: &token_prefix, + }; + + assert!(can_use_datatable_role(&tenants(&["u/alice"]), &authed)); + assert!(can_use_datatable_role(&tenants(&["g/analysts"]), &authed)); + assert!(can_use_datatable_role(&tenants(&["f/finance"]), &authed)); + assert!(can_use_datatable_role(&tenants(&["*"]), &authed)); + assert!(!can_use_datatable_role(&tenants(&[]), &authed)); + assert!(!can_use_datatable_role( + &tenants(&["u/bob", "g/ops"]), + &authed + )); + // A bare name is not a principal: only the three prefixes and the wildcard match. + assert!(!can_use_datatable_role(&tenants(&["alice"]), &authed)); + + // An admin of the governing workspace reaches every role: they can edit the lists anyway. + let is_admin = true; + let admin = crate::db::AuthedRef { is_admin: &is_admin, ..authed }; + assert!(can_use_datatable_role(&tenants(&[]), &admin)); + } + + #[cfg(not(all(feature = "private", feature = "enterprise")))] + #[test] + fn without_the_enterprise_edition_no_tenant_list_covers_anyone() { + let groups = vec![]; + let folders = vec![]; + let scopes = None; + let token_prefix = None; + let is_admin = true; + let is_operator = false; + let admin = crate::db::AuthedRef { + email: "alice@windmill.dev", + username: "alice", + is_admin: &is_admin, + is_operator: &is_operator, + groups: &groups, + folders: &folders, + scopes: &scopes, + token_prefix: &token_prefix, + }; + assert!(!can_use_datatable_role(&tenants(&["*"]), &admin)); + assert!(!can_use_datatable_role(&tenants(&["u/alice"]), &admin)); + } + + #[test] + fn a_datatable_ref_splits_off_its_role() { + assert_eq!(parse_datatable_ref("sales").unwrap(), ("sales", None)); + assert_eq!( + parse_datatable_ref("sales?role=analytics").unwrap(), + ("sales", Some("analytics")) + ); + // The key matches case-insensitively, the way the `-- role` annotation does. + assert_eq!( + parse_datatable_ref("sales?Role=analytics").unwrap(), + ("sales", Some("analytics")) + ); + + // A query string that does not parse is refused rather than read as "no role": resolving + // it to the data table's default would connect as a login the caller never asked for. + for malformed in [ + "sales?role=", + "sales?role=an;alytics", + "sales?x=1&role=analytics", + "sales?role=a&role=b", + ] { + assert!( + parse_datatable_ref(malformed).is_err(), + "silently ignored: {malformed}" + ); + } + } + + #[test] + fn an_entry_owns_a_database_or_points_at_one_but_never_both() { + let terminal = DataTable { + database: Some(DataTableDatabase { + resource_type: DataTableCatalogResourceType::Instance, + resource_path: "dt_main".to_string(), + }), + reference: None, + governed_by: None, + forked_from: None, + migrations_enabled: None, + permissions: None, + }; + assert!(validate_datatable_shape("main", &terminal).is_ok()); + + let pointer = DataTable { + database: None, + reference: Some(DataTableReference { + workspace_id: "prod".to_string(), + datatable: "main".to_string(), + }), + ..terminal.clone() + }; + assert!(validate_datatable_shape("main", &pointer).is_ok()); + + let both = DataTable { database: terminal.database.clone(), ..pointer.clone() }; + assert!(validate_datatable_shape("main", &both).is_err()); + + let neither = DataTable { database: None, reference: None, ..terminal.clone() }; + assert!(validate_datatable_shape("main", &neither).is_err()); + } + #[test] fn test_parse_fork_branch() { // Generated fork (`wm-fork-abc`) and dev workspace (`staging`) forms. @@ -2421,6 +3580,7 @@ mod tests { webhook_secret: None, webhook_url: None, webhook_error: None, + enabled_by: None, last_synced_sha: synced .iter() .map(|(r, s)| (r.to_string(), s.to_string())) @@ -2785,3 +3945,17 @@ pub async fn dbt_warehouse_resource( .map(|t| t.to_string()); Ok((path, target)) } + +#[cfg(test)] +mod guest_allowance_tests { + use super::*; + + #[test] + fn guest_seats_round_up_past_the_allowance() { + assert_eq!(guest_seats(0), 0); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW), 0); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW + 1), 1); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW + GUESTS_PER_SEAT), 1); + assert_eq!(guest_seats(FREE_GUESTS_PER_WINDOW + GUESTS_PER_SEAT + 1), 2); + } +} diff --git a/backend/windmill-common/tests/billing_workspace.rs b/backend/windmill-common/tests/billing_workspace.rs index 1b0186a5cf..3025061249 100644 --- a/backend/windmill-common/tests/billing_workspace.rs +++ b/backend/windmill-common/tests/billing_workspace.rs @@ -3,7 +3,7 @@ use sqlx::{Pool, Postgres}; use windmill_common::workspaces::{ - count_paid_seats, count_workspace_forks, fork_chain_depth, fork_subtree_height, + billable_seats, count_paid_seats, count_workspace_forks, fork_chain_depth, fork_subtree_height, get_billing_workspace_id, invalidate_billing_workspace_cache, list_fork_descendants, }; @@ -106,11 +106,17 @@ async fn paid_seats_and_fork_count(db: Pool) { insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await; insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await; insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await; - // These must NOT count towards seats. + // These must NOT count towards seats. The service account is a non-operator, so counting it + // would inflate the developer tally the invoice line is written from, not the operator one. insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await; insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await; assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3); + let breakdown = billable_seats(&db, "seat-root").await.unwrap(); + assert_eq!( + (breakdown.developers, breakdown.operators, breakdown.seats), + (2, 2, 3) + ); insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; diff --git a/backend/windmill-common/tests/dbt_graph_storage.rs b/backend/windmill-common/tests/dbt_graph_storage.rs index 7b4971312a..3226d27e9e 100644 --- a/backend/windmill-common/tests/dbt_graph_storage.rs +++ b/backend/windmill-common/tests/dbt_graph_storage.rs @@ -7,9 +7,10 @@ use sqlx::{Pool, Postgres}; use windmill_common::dbt_manifest::{ - clear_dbt_editor_graphs, clear_dbt_manifest_version, prune_dbt_run_graphs, - replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode, - DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, + clear_dbt_editor_graphs, clear_dbt_manifest_version, clear_dbt_script_state, + clear_dbt_script_state_if_path_retired, move_dbt_script_state, prune_dbt_run_graphs, + replace_dbt_editor_graph, replace_dbt_manifest, IngestedColumnEdge, IngestedManifest, + IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, }; const WS: &str = "test-workspace"; @@ -52,10 +53,36 @@ fn manifest(names: &[&str]) -> IngestedManifest { .windows(2) .map(|w| (format!("model.p.{}", w[0]), format!("model.p.{}", w[1]))) .collect(), + // Same reason: a project that opted into the analysis pass has these, and + // a fixture without them leaves every column-edge insert and sweep in + // this file unexecuted. + column_edges: names + .windows(2) + .map(|w| IngestedColumnEdge { + parent_unique_id: format!("model.p.{}", w[0]), + parent_column: w[0].to_string(), + child_unique_id: format!("model.p.{}", w[1]), + child_column: w[1].to_string(), + lineage_kind: "copy".to_string(), + }) + .collect(), ..Default::default() } } +/// Column edges of one version, so the sweeps can be shown to reach them. +async fn column_edges_for(db: &Pool, hash: i64) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + WS, + hash + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} + /// Edges for one version, so a test can assert the batched insert ran at all. async fn edges_for(db: &Pool, hash: i64) -> i64 { sqlx::query_scalar!( @@ -135,6 +162,33 @@ async fn an_identical_run_stores_no_snapshot(db: Pool) { assert_eq!(markers(&db, 1).await, 1, "and leaves no marker of its own"); } +/// A column that is projected AND used as a predicate for the same output column +/// has both a `copy` edge and a `scan` one. They are two facts, and the digest +/// counts both — so the uniqueness key has to carry `lineage_kind`, or the +/// second is dropped by `ON CONFLICT DO NOTHING` while the digest still claims +/// it was stored. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn both_kinds_of_one_column_pair_are_stored(db: Pool) { + deploy_script(&db, 1).await; + let pair = |kind: &str| IngestedColumnEdge { + parent_unique_id: "model.p.a".to_string(), + parent_column: "id".to_string(), + child_unique_id: "model.p.b".to_string(), + child_column: "id".to_string(), + lineage_kind: kind.to_string(), + }; + let mut m = manifest(&["a", "b"]); + m.column_edges = vec![pair("copy"), pair("scan")]; + + let mut tx = db.begin().await.unwrap(); + replace_dbt_manifest(&mut tx, WS, PATH, 1, None, &m, "root") + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!(column_edges_for(&db, 1).await, 2, "both kinds survive"); +} + /// A run whose model set differs keeps its own, and the version's is untouched: /// this is what lets an older run page render the project that run built. #[sqlx::test(migrations = "../migrations", fixtures("base"))] @@ -235,7 +289,11 @@ async fn clearing_one_version_leaves_the_others(db: Pool) { // this is where two versions coexist: it pins the batched edge insert // against a real database as well as the version scoping. assert_eq!(edges_for(&db, 1).await, 0, "the cleared version's edges go"); - assert_eq!(edges_for(&db, 2).await, 1, "the other version keeps its own"); + assert_eq!( + edges_for(&db, 2).await, + 1, + "the other version keeps its own" + ); } /// The routes that hard-delete a path clear no graph rows: they delete the @@ -275,6 +333,8 @@ async fn deleting_the_script_cascades_to_every_sidecar(db: Pool) { assert_eq!(nodes_for(&db, 2, DEPLOYED_GRAPH).await, 0); assert_eq!(edges_for(&db, 1).await, 0); assert_eq!(edges_for(&db, 2).await, 0); + assert_eq!(column_edges_for(&db, 1).await, 0); + assert_eq!(column_edges_for(&db, 2).await, 0); assert_eq!(markers_for_path(&db).await, 0); } @@ -315,7 +375,7 @@ async fn the_sweep_takes_old_snapshots_and_spares_the_version(db: Pool tx.commit().await.unwrap(); // Age one snapshot past the window, rows and marker together. - for t in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for t in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "UPDATE {t} SET ingested_at = now() - interval '400 days' WHERE job_id = $1" )) @@ -363,7 +423,11 @@ async fn only_the_newest_deploys_keep_their_graph(db: Pool) { // The newest is always among them: losing the live version's graph would // empty the page of every run of it. assert_eq!(nodes_for(&db, over, DEPLOYED_GRAPH).await, 1); - assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0, "the oldest is reclaimed"); + assert_eq!( + nodes_for(&db, 1, DEPLOYED_GRAPH).await, + 0, + "the oldest is reclaimed" + ); } /// The third provenance: a `parse` of the EDITOR's buffer, which names no @@ -480,17 +544,27 @@ async fn a_version_clear_spares_editor_graphs_and_a_path_clear_does_not(db: Pool replace_dbt_editor_graph(&mut tx, WS, PATH, job, ME, &manifest(&["a"]), "root") .await .unwrap(); - clear_dbt_manifest_version(&mut tx, WS, PATH, 1).await.unwrap(); + clear_dbt_manifest_version(&mut tx, WS, PATH, 1) + .await + .unwrap(); tx.commit().await.unwrap(); assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0); - assert_eq!(editor_nodes(&db, job).await, 1, "the buffer's graph survives"); + assert_eq!( + editor_nodes(&db, job).await, + 1, + "the buffer's graph survives" + ); let mut tx = db.begin().await.unwrap(); clear_dbt_editor_graphs(&mut tx, WS, PATH).await.unwrap(); tx.commit().await.unwrap(); - assert_eq!(editor_nodes(&db, job).await, 0, "retiring the path takes it"); + assert_eq!( + editor_nodes(&db, job).await, + 0, + "retiring the path takes it" + ); } /// A preview names its own PATH and needs only `jobs:run`, so a bound over the @@ -552,3 +626,157 @@ async fn editor_markers(db: &Pool) -> i64 { .unwrap() .unwrap_or(0) } + +/// A deferral resolves a `ref()` through the manifest of the last successful run +/// at this path, so that state has to follow the script the way the retry state +/// does: a rename must not strand it, and a path no live dbt version occupies +/// must not hand its manifest to whatever is created there next. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn environment_state_follows_the_script(db: Pool) { + const MOVED: &str = "f/test/renamed"; + deploy_script(&db, 1).await; + publish_environment_state(&db, PATH).await; + + let mut tx = db.begin().await.unwrap(); + move_dbt_script_state(&mut tx, WS, PATH, MOVED) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, PATH).await, 0); + assert_eq!(environment_states(&db, MOVED).await, 1); + + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state(&mut tx, WS, MOVED).await.unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, MOVED).await, 0); +} + +/// Archiving or deleting ONE version must not take the path's state with it — +/// the live version's next deferral still needs it — while the last one leaving +/// must, or a script later created at that path inherits the previous project's +/// manifest. The condition is a `NOT EXISTS` in raw SQL, so both directions are +/// pinned against a real database. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn state_goes_only_once_no_live_version_is_left(db: Pool) { + deploy_script(&db, 1).await; + deploy_script(&db, 2).await; + publish_environment_state(&db, PATH).await; + + retire(&db, 1).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 1, + "another version is still live here" + ); + + retire(&db, 2).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 0, + "the last one leaving takes it" + ); +} + +async fn retire(db: &Pool, hash: i64) { + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + hash + ) + .execute(db) + .await + .unwrap(); +} + +/// The worker publishes under a guard naming the version that ran, and the whole +/// point of it is a job that finishes late: its script can be renamed away and an +/// unrelated one created at the same path while it runs, and that project must +/// not inherit this one's manifest as its deferral state. Enforced in raw SQL, +/// where a refactor can drop a predicate with no type error, so it is pinned +/// against a real database — the same shape `dbt_state::publish` issues. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_late_job_cannot_publish_for_a_path_it_no_longer_owns(db: Pool) { + deploy_script(&db, 1).await; + assert_eq!(guarded_publish(&db, PATH, 1).await, 1, "its own version"); + assert_eq!( + guarded_publish(&db, PATH, 2).await, + 0, + "a version that never lived here" + ); + + // The script is gone from this path and another one takes it. + sqlx::query!( + "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + WS + ) + .execute(&db) + .await + .unwrap(); + deploy_script(&db, 3).await; + assert_eq!( + guarded_publish(&db, PATH, 1).await, + 0, + "the late job's version does not own this path any more" + ); +} + +/// The predicate `dbt_state::publish` locks the script row on, reduced to what it +/// decides. Keep the two in step — this file cannot call `publish` itself, which +/// is `pub(crate)` in `windmill-worker`. +async fn guarded_publish(db: &Pool, path: &str, ran: i64) -> u64 { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text + WHERE EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false + AND language = 'dbt' + AND (hash = $4 OR $4 = ANY(parent_hashes))) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + WS, + path, + uuid::Uuid::from_u128(9), + ran, + ) + .execute(db) + .await + .unwrap() + .rows_affected() +} + +async fn publish_environment_state(db: &Pool, path: &str) { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + WS, + path, + uuid::Uuid::from_u128(9), + ) + .execute(db) + .await + .unwrap(); +} + +async fn environment_states(db: &Pool, path: &str) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_private.pem b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_private.pem new file mode 100644 index 0000000000..bb443cf6bf --- /dev/null +++ b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_private.pem @@ -0,0 +1,100 @@ +-----BEGIN PRIVATE KEY----- +MIISQgIBADANBgkqhkiG9w0BAQEFAASCEiwwghIoAgEAAoIEAQC6deMC6KWwkWV2 +9xb4bXLt6ceNGQD3gJH3sYqpulWsZB0L3TdqXwnzL3C/8Uf3K8hLZ4Tw+llHCxaS +Z7aTEYyeLDJuYuJPVS8DAgVnXDluoDg7B1UtvoUa1/MjLQm4f/Qp322i7l0SH9kx +NRb7UZp/xx1tL/kiq9NIv1inK4ZC0DbleybBOoLlwP9Zqhazw3MJdZAyey4FAXTx +bZcjG3UtiZ2COXVP1qdtqhLrQQQsicoLqZuxAfXdIU6xgcYgGapVQNTVbo7w8pHn ++6RzlPb9rqyBs2IiKLFSb467mtZeM9dfbyD4DICy42O/JKAbTZw/f3trbs+tu+sK +rj7ysk+lGzuXj2/irOuvOSbzpxERdZuDyLOhlMej5+aJXkrmVhVPf/3wt/6oKCzo +7Ta24OvGaz6pxK1iDOQvzkCqJpaTjSfpcscHyU3ztXCUINtsq7s78XoOxJZkerfq +LDxuutYviFiB9pQDzYdErL8XMUkRXXY22hpnLHt8GV+lNCJs2KxcECzkgwfjIn9X +Y66U4iO+k9EvY/FHyI25ur0eKWDnJAPL6PypVhx9NOuBkot6nwUugzWzM/MKJ9Hf +1n9pwVNlMam8zgMQ5Ivu3+RiOV3P0iuAaw7d1sZRSSX11l/EyjGFDpwmnFNjleEK +vnop6rCftWAsd2XJHXiD1/80QIgKJHdflpHOcy6Su5w/CRYc/jC2rJjUr9W+hBwg +fnayip8MTMa4cAXzbFmSc5NCzSsqHkVq/4RdfdMephubGmWcPDQnkUf/n/RvoZNi +VktW34lX3YIe97600vSWPfWahj/EUnfI8m+4Qzuz2o6N9basCNftKPDIc5ej3PMS +RVIgBhEXUh1wVFdEUs8RKHg/QV0xrN2qbKcS8XvV3fFfLuFZo4a66nZ5zgGGJUgL +ED41cEqf/sQObfhGgIw25v/315ARiv9thRF3tUMLGQZNQ8tSs2VMuU6szBQciplx +y2y4JzXdv+G0PvheE7tVdsGsMipSHTZDdNZo51MOBSgsNTyNmWMZlsfoZTTZ2o2y +XtKv8xUP01KnlwR6Z8hY1CAPxM1QKDU5nKp9T/df6GEhohps9gSt6cs6SHlH8piq +pyR0xiDvS+vcPN/IbOaRgE9WmfM5py72SNZbetiUHdUTmAC1dd0IH3opte1uVpQQ +sauKVWZuzV8Bcd2O06QFOVJDwoxm00YaF0rqhaisppah5eQMN/ctlqWVDHXR8laV +CNkzWYE/jxAU9dGuCMCIZ7BSs5U9mZ7pWstagXWEFTI9IugjvNnaZDijZQJSPmAZ +gQtAxH/z8R9e/PihW9dDQc/nKBC9Lpta3oFGq74fc7Y+Wu/Uskjf8qtFOOVrvfZx +kwlRR0s5AgMBAAECggQAEEeerHBXlxT6wOpN7R7RPAh/TchzFyj4laHAs5T7SfPr +vU9xOS5XFg58T6XnXKNL95lwOAIDrXpAXEQF4s1Tc7P0K0Mi8z5CvoZ8pkudEBWZ +Ez5Op5HM2U0lg1q5UIyLauSBFAqvwZdaRTuXaBX14VYnMZNQs0BBUEa0nNPmAwte +36Zi+df7+cdiBBgfp05XSuoizuWGXRWFelbRLVWGY5VyVR5jKbdaIu8Ie9XPYXwe +UepbF5dOOiJgVos4GzFid9MTQhdHD65+Ir0L3LOS5Q9NLH3XAZh/8uyej66zLMJl ++YUgYcflrLjByryN5hReogSlZ/ni1A5mNSaFhHchEQ9Roqf5YtcqNMBOsFNT4Cxe +iaEKXf85dwwWSsQa0gWOEsTgDRZje5cWZLfhYRI+Q7K1Y+kyIi9x9OKTLsfzuapH +Y9CLpBhlX/Plgmlj/xb3YAJzX7q2TsuQJzf5hsl1Te0fnSlEPxjAyLQ0cvfwoz1/ +xsZAK+XsWid0/F2S6+H3GoqnAMSX+bLXX2/qnEEqUcdiKjsVo+jUxvSQ8oM2XLf+ +RN1wwfAsyJnIPiXnW6tqrmbyV1Uf0TEs19hIg5lHiLxPT5bFGcJ6nT6bGCBeGnkF +J08xmxjXc91pHQ+8iNgGftGJvK1FcppxbZi33dUJ2sfhP8Ck+GxES26eb5d6yTfD +oZRQmESvikHPU5shm+YbjPrYhEbEgLZKO3wUhCAvoFLN2TG0i2v//aaCSI/XfNWt +1XyE+uN/o6kQEPvaUy/0uHBuzoC4pZ8or22ywmeftCOHuT9YE9LIJuU81qMJvjP7 +BJ6YkcTO6G89iD0g6HHYRiD9I+pgih8B1NPq4VmFuJrvaqgAX9srmmMt7FXBrcqa +JSyLS+WeHTLlOiHN1DqOgpgq0gq6vC1i53xPqSUkFzAKpDOWgGNMVAABFN9EOVmM +zvPnJThz5PHDAuud9eaLjZ0V76IVXmRTz22warkdKhBW5BzDzy+zNp8mF5SMH9mZ ++JQvCGTc2itiKGKMDzvY99pBzHRlrlFL7n5SHLqxclBfnEJcU9wPvFyZ+6k8MGim +0kFrK63NqiceNWp5vdU/6V+boCYoFG8PDbABUxCbb+lhtnYx5AUcKfKX74Uv4gpX +AM0gsaY8U+4ATJbgoCTqplTKvXtyw4zcpJAXCMnhygKyDKNOk+Lckfquwx9r/TQN +hbbjjvwgBkzJtAlylCVYYEJ/03lazRpGR5osCksECxNU4TMDRD2Hd+zwUxdeRunm +7EusRSrKq8swi8LFyV7E8ISwwXegqVarHqXjCHhcn/5Ozw6PkVk39Ug2UE4ibesC +MzVP3lfP67gXgGCzYrImIUuqHGTyLFs8/B8+2OmdyQKCAgEA6PCozdeeQS0P/+ah +E015KECxB0aGbrvdpczXX5szHYb8KhejvHmVV2OLFisL7GyMeBhEZXAeKtMZwd66 +FMORgdFACpckR/a/aVEnebHx5fOKtAQfhLLH/3YE5dFIWxu9SKNZBrX0lIBFXfoK +lZYM7cGm7vN55AK99bYfCAqFs+HI9sDQLjbaxR41UOfZwPjA9VjBQ1vBr0N3UoOT +xf8Rw6hlAd+ED7iJavn6KUb64D5eB4eGSZNXM5HCp2ZtUFN/4bU5lbHPCSK29+Wd +mM5flZtIKQ7PzGyXqv4Kbtz9o1hPOQPeH0faDEcYJLHV/LAxHc1AkqJDXU8HANhI +m/XI44RlJesNntOdJuCumHtTAsreWAYoydXurOWHkDIpx6OLi0F3eJrOV6dhwCCG +v3bX4PIBjYZKyfHukcoCzGetXpu5bDKC1uwwZLJmF/QTO3AeVpWNrbWkBfj/PCio +8RodjVlPkDaoCrDmpca22fNo+93IM2m1SWdSnTlXvgg58cLUrlMUbg24Lo1cyySF +1AnRw0uvaFLdH8iMtKZIA3zgCZyUKV1DcGUKsdcL/9IYD6EO5bCcA6p5gcxxBOan +6MEHIuJKycmZMGutN5rm3uDHQgmNCs25ma3fWQF0s/dRJ2lg947qKUKevqBkd4gt +BuB8VXUGsHXGDWHfz/eLjWYA3k8CggIBAMzrTqnmT27U1SXPwEXI0kc6A44D8skD +a4rqxOMBTGvu+HeVdEBMcxYnGeJV2QtZY96KXwadiWuqdEr4Pc7JWnAa2f9yWNCb +QI0YJSWSNcDuPsSV1K15MM7uxKk/oNcXdbjrAfVLzCmEYDg085QV80rl7Q6WRaYv +acvipeZY82LuGhtAJAN6S8VWmDqZ5vBDd61F09AlBrpIFoSBRnYVkEwaXXwFJauz +S+kEmCIb97tBf7nr08ldoNvDdecRtm+s9odfUhm+ONroZf7OEyTCOXB666oDogpI +hktuliNUVpXeZ5QtkzpxP2SBbHNWZ9LiAGpTv2wa7z7k1PqCVR6ABGMkTkge41iy ++v82KYi4qRnUfVlFcn9xLyBDaaCWJYrsux8OmAW48cVN1Og0tKu/T+bFUvHs0etZ +oijYoYZaf9ERO0X0tuMXKIuT0QcPnkcT+KXFY6uzOC1UmF5ITCeKogGwzkFbvInA +w8OiFYBkgXZ41tE/oOARm1AhuaNc/ZfWujLECjuJgvw0uhdUyihGw8ZJv91WiIjX +Jqq1BNrhvDQqTPSnDvhcBBDGdQA6HJRGAerIx7PgCKzS2WbrfmsFzjfZ3zbrUSNf +e8CM37aDzte25g7HmvFuwOLmE9DXNnE+cWGpb/S1l4ACahnqKF9zj3uKVpN1kG+l +xFH+ADFgNCP3AoICAQDW+fuf9QVdezwP3IX9bXuKVaHLaB/7JDBbqt99oZsb/g7M +vgJjmAXmn7ymGdYoPMmqsCK3RzDgLvDPhkFPlmm8WCR1YaWfMK7YgIgg/MOViB0T +I9b+vYxsTDwa6PmQ89nNpssrsVNmlbbvEysfgFqwNg9c6aEteAZeNNUQs71PFOpG +KA45CHEAiG+GGmNRf4F9LIs4If9vQ6Fkqn//s/ZuDwQhJWoYpCNWchuYaqRvvb9D +zaR2VJ+evT/bBGGQR293G6xoZa//62TXBbdbZeK7DJMdKUlHjVSv/0IbrKH8VSzQ +HqypS8OFnzs96k3lpam+qkWUzmHDRILfGbxxCvV6XB6JTxPD5+JaFfOsT+x9YEKx +FAYit+hM8owY3OLIFzxaqXGcAaq7JIPHL32MSB7VwCMEvNJa64dUTw4kHO7YqHkI +zNkJhvLmi+pC4wO40DF/kQgvL7qPtIvBtnisAoERiUIvgvG05CwJBIqOPOE/yR/R +qxYfB1csONasT6VCZJ+pvozVQJ4hgF8KL8u+s80+IqRgcFEY2xpYQjml1mJhHHt0 +bVkm6mljVWcCkp+YFPjNrB9gFr2Kt+X6MvD1kueGHzgMYJybHlS/Q5scSHA8ltnv +BYvrs5az6u8MhAwGhm9yKRgcOhqNLTNrSWbcuOKmb5lb02WTxicrJHdLaL8WawKC +AgAvXHdK51MWj5RP7+tFdbu8AJmPujPj56l0avxkZwltrMa6hM1giwrSZOkjmanX +aKqfZaW+1qxLORM1vJg0ce84F8yosJn8ZNWPhrQXMerAk4/oSaTlDyyLtz22zGD1 +NZPc0FwcgI9ITTMb6okdgdFAT1Hx3CNu3Gh8Wf7z9T0TcWqRP8rTDvHyQyhd6vIo +N7+ySYBCDDy40y55I5PQeP1nyXQv1NS3EXKLAafTwlkIa4QEyY+ia8Rg5pUaecqL +lxNHiG1mS8PCPJJOSPy9szIpYA+VtM8/xzFav81tjvGQI94Wrk3n5QK8zPju01qd +r+RersqARt1i0Oq0l2jq6rWY4GxDWPg8qOlAERmaYuuab7Uk69shv1wf/dSs6+bz +a/F3pRzXvb4LVwjNPlk/lLD36kVYat69oXLMM/1pGwONo4jqizu10sVpNRnLov12 ++hOu5Kc7qOlj+LhamiUXwn/dGishif1SbT4RuiV0DkapxhYIOMeK4ibeZ10Ld9OY +w2U3lH208vxWJC2adNtgYbKf4q24j2ZxEiD6tWnQ2PlxqREuPmpEVqgN5IBoNAYk +8uVAiXU2iqIWp4jds7H9NFNML0va2ppfLZNhlm2iNmaZDsmvQsz53howlvKfgw4v +MDpuUwXwLgQ87tHs+WSpYwEXkYqe/D4Mv+hW9vi+cLi8UwKCAgAsehP6DOrt9bA3 +pVdtHJG1a9fm/rXC9TQS2oSE2paXRZpxhbk917Gdl2njo7tnoGXXBpQOb51/HUXX +QTARdp2LP1VFn9RJP3CwaK+Sxs1IBa9r1/p2ePDR85QR1HxN+Gj98EcEfKl1RSe6 +RmLOkLIg4pkNrDhwWyYJ1H5//d/yUQlZ015fQAKaHCdaoz7L3+0jOyFXDGrDr+Gc +680SyVqPfRx5dAm8TrfyXW6Q2uZEhIqxbcxTdG98Pxi0qLLAS/nD16IHC1iljHsI +QrfUvFvGg5v7kxeXrP0VPMRM0/Y8vno3Bbi04I0Pk3Wko8UhWuiUMjs+TVbEPKox +8zG/G+R5EpHKDxNyfUUwZ9+lyjFmBZOcVd3dze23VQYCzV/W04yiQdEl2C/sED2m +NycJ3GN/ByqpJJZY3u+B/WAiHPLe1ivIEeIjwdBt2w/3L/rKoRBR/P8LSRXLIIh4 +Xk1BMNygJxHTwsvFGhflx89+1BO/n8HrR0s/2G1QanxGdgso9bwmXMDrmJ+XxrUY +qBmDCU8RkBdUHKOuJlWWn2JaIswsEE5kAk7MgNNaBTv91VBYCDs50NW8VBaKnJjN +j+hXI80qKi7WqeoonFKFLDgUYbKnmzKsX+T0x/H99v+7Or5PmAPN+hRwj54/LEVR +MsjrTlnoCGXKgPQQ5ItWviGjqoCTiQ== +-----END PRIVATE KEY----- diff --git a/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_public.pem b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_public.pem new file mode 100644 index 0000000000..44645f5af2 --- /dev/null +++ b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_public.pem @@ -0,0 +1,25 @@ +-----BEGIN PUBLIC KEY----- +MIIEIjANBgkqhkiG9w0BAQEFAAOCBA8AMIIECgKCBAEAunXjAuilsJFldvcW+G1y +7enHjRkA94CR97GKqbpVrGQdC903al8J8y9wv/FH9yvIS2eE8PpZRwsWkme2kxGM +niwybmLiT1UvAwIFZ1w5bqA4OwdVLb6FGtfzIy0JuH/0Kd9tou5dEh/ZMTUW+1Ga +f8cdbS/5IqvTSL9YpyuGQtA25XsmwTqC5cD/WaoWs8NzCXWQMnsuBQF08W2XIxt1 +LYmdgjl1T9anbaoS60EELInKC6mbsQH13SFOsYHGIBmqVUDU1W6O8PKR5/ukc5T2 +/a6sgbNiIiixUm+Ou5rWXjPXX28g+AyAsuNjvySgG02cP397a27PrbvrCq4+8rJP +pRs7l49v4qzrrzkm86cREXWbg8izoZTHo+fmiV5K5lYVT3/98Lf+qCgs6O02tuDr +xms+qcStYgzkL85AqiaWk40n6XLHB8lN87VwlCDbbKu7O/F6DsSWZHq36iw8brrW +L4hYgfaUA82HRKy/FzFJEV12NtoaZyx7fBlfpTQibNisXBAs5IMH4yJ/V2OulOIj +vpPRL2PxR8iNubq9Hilg5yQDy+j8qVYcfTTrgZKLep8FLoM1szPzCifR39Z/acFT +ZTGpvM4DEOSL7t/kYjldz9IrgGsO3dbGUUkl9dZfxMoxhQ6cJpxTY5XhCr56Keqw +n7VgLHdlyR14g9f/NECICiR3X5aRznMukrucPwkWHP4wtqyY1K/VvoQcIH52soqf +DEzGuHAF82xZknOTQs0rKh5Fav+EXX3THqYbmxplnDw0J5FH/5/0b6GTYlZLVt+J +V92CHve+tNL0lj31moY/xFJ3yPJvuEM7s9qOjfW2rAjX7SjwyHOXo9zzEkVSIAYR +F1IdcFRXRFLPESh4P0FdMazdqmynEvF71d3xXy7hWaOGuup2ec4BhiVICxA+NXBK +n/7EDm34RoCMNub/99eQEYr/bYURd7VDCxkGTUPLUrNlTLlOrMwUHIqZcctsuCc1 +3b/htD74XhO7VXbBrDIqUh02Q3TWaOdTDgUoLDU8jZljGZbH6GU02dqNsl7Sr/MV +D9NSp5cEemfIWNQgD8TNUCg1OZyqfU/3X+hhIaIabPYErenLOkh5R/KYqqckdMYg +70vr3DzfyGzmkYBPVpnzOacu9kjWW3rYlB3VE5gAtXXdCB96KbXtblaUELGrilVm +bs1fAXHdjtOkBTlSQ8KMZtNGGhdK6oWorKaWoeXkDDf3LZallQx10fJWlQjZM1mB +P48QFPXRrgjAiGewUrOVPZme6VrLWoF1hBUyPSLoI7zZ2mQ4o2UCUj5gGYELQMR/ +8/EfXvz4oVvXQ0HP5ygQvS6bWt6BRqu+H3O2Plrv1LJI3/KrRTjla732cZMJUUdL +OQIDAQAB +-----END PUBLIC KEY----- diff --git a/backend/windmill-common/tests/notify_events.rs b/backend/windmill-common/tests/notify_events.rs index 8161130875..285086308e 100644 --- a/backend/windmill-common/tests/notify_events.rs +++ b/backend/windmill-common/tests/notify_events.rs @@ -360,6 +360,77 @@ async fn test_trigger_notify_workspace_key_change(db: Pool) { ); } +/// The address a job runs as is served from a process-local cache, so every change that can move +/// a `(workspace, username)` -> email mapping has to reach the other replicas as an eviction. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_trigger_notify_user_email_change(db: Pool) { + let before_id = get_latest_event_id(&db).await.unwrap(); + + sqlx::query("UPDATE usr SET email = 'renamed@windmill.dev' WHERE workspace_id = 'test-workspace' AND username = 'test-user'") + .execute(&db) + .await + .expect("Failed to change email"); + + let events = poll_notify_events(&db, before_id) + .await + .expect("Should poll events"); + assert!( + events.iter().any(|e| e.channel == "notify_user_email_change" + && e.payload == "test-workspace:test-user"), + "email change should evict the key it moved" + ); + + // A superadmin outside their workspaces resolves through `password`, which names no + // workspace: the wildcard is the only way to reach that key. `super_admin` is half of what + // that fallback matches on, so losing it moves the mapping just as the address does. + for (label, stmt, expected_aliases) in [ + ( + "email change", + "UPDATE password SET email = 'sa2@windmill.dev' WHERE email = 'test@windmill.dev'", + // old address, new address, and the username that outlives both + vec!["test@windmill.dev", "sa2@windmill.dev", "test-user"], + ), + ( + "demotion", + "UPDATE password SET super_admin = false WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ( + "promotion", + "UPDATE password SET super_admin = true WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ( + "deletion", + "DELETE FROM password WHERE email = 'sa2@windmill.dev'", + vec!["sa2@windmill.dev", "test-user"], + ), + ] { + let before_id = get_latest_event_id(&db).await.unwrap(); + sqlx::query(stmt) + .execute(&db) + .await + .unwrap_or_else(|e| panic!("Failed to apply superadmin {label}: {e}")); + + let events = poll_notify_events(&db, before_id) + .await + .expect("Should poll events"); + // Every alias the principal can be spelled as, since `resolve_username_to_email` + // matches a `u/` principal against the username or the address. + let evicted: Vec<&str> = events + .iter() + .filter(|e| e.channel == "notify_user_email_change") + .filter_map(|e| e.payload.strip_prefix("*:")) + .collect(); + for alias in expected_aliases { + assert!( + evicted.contains(&alias), + "superadmin {label} should evict {alias}, got {evicted:?}" + ); + } + } +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_trigger_notify_token_invalidation(db: Pool) { // First insert a session token with token_hash and token_prefix diff --git a/backend/windmill-common/tests/permissioned_as_authz.rs b/backend/windmill-common/tests/permissioned_as_authz.rs new file mode 100644 index 0000000000..25c219845f --- /dev/null +++ b/backend/windmill-common/tests/permissioned_as_authz.rs @@ -0,0 +1,81 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::auth::fetch_authed_from_permissioned_as; + +/// The address handed to `fetch_authed_from_permissioned_as` may come from a cache that a +/// username reassignment has outrun. It must not be believed: the workspace role is keyed on the +/// principal while `super_admin` and `email_to_igroup` are keyed on the address, so trusting a +/// stale one would run the new holder's job with the previous holder's instance privileges. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_stale_address_cannot_carry_the_previous_holders_privileges(db: Pool) { + // `test-user` in the fixture is a superadmin with the address `test@windmill.dev`. Free the + // username and hand it to somebody who is not, exactly as an offboard-then-onboard would. + sqlx::query("DELETE FROM usr WHERE workspace_id = 'test-workspace' AND username = 'test-user'") + .execute(&db) + .await + .expect("free the username"); + sqlx::query( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('newcomer@windmill.dev', 'x', 'password', false, true, 'Newcomer')", + ) + .execute(&db) + .await + .expect("create the new account"); + sqlx::query( + "INSERT INTO usr(workspace_id, email, username, is_admin, role) + VALUES ('test-workspace', 'newcomer@windmill.dev', 'test-user', false, 'User')", + ) + .execute(&db) + .await + .expect("reassign the username"); + + // What a replica that has not yet consumed the eviction would pass: the principal is the + // reassigned username, the address is the one it cached for the previous holder. + let authed = fetch_authed_from_permissioned_as( + "u/test-user", + "test@windmill.dev", + "test-workspace", + &db, + ) + .await + .expect("should authenticate the current holder"); + + assert_eq!( + authed.email, "newcomer@windmill.dev", + "the principal's live address must win over the one supplied" + ); + assert!( + !authed.is_admin, + "the new holder must not inherit the previous holder's superadmin" + ); +} + +/// A disabled member still holds its username in the workspace. Workspace usernames are only +/// unique per workspace, so an unrelated instance superadmin can share it, and falling through to +/// the `password` fallback would run the disabled member's jobs as that superadmin. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_disabled_member_never_resolves_to_a_same_named_superadmin(db: Pool) { + sqlx::query( + "UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace' AND username = 'test-user-2'", + ) + .execute(&db) + .await + .expect("disable the member"); + sqlx::query( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('other-superadmin@windmill.dev', 'x', 'password', true, true, 'Other', 'test-user-2')", + ) + .execute(&db) + .await + .expect("create the same-named superadmin"); + + for supplied in ["test2@windmill.dev", "other-superadmin@windmill.dev"] { + let authed = + fetch_authed_from_permissioned_as("u/test-user-2", supplied, "test-workspace", &db) + .await; + assert!( + authed.is_err(), + "a disabled member must not authenticate (supplied {supplied}): {:?}", + authed.map(|a| (a.email, a.is_admin)) + ); + } +} diff --git a/backend/windmill-common/tests/queue_metrics_series.rs b/backend/windmill-common/tests/queue_metrics_series.rs new file mode 100644 index 0000000000..f6bc7b4056 --- /dev/null +++ b/backend/windmill-common/tests/queue_metrics_series.rs @@ -0,0 +1,136 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::queue_metrics::{read_queue_metrics_series, QUEUE_METRIC_STALE_SECS}; + +const WINDOW: f64 = 3600.0; + +/// Store a sample written `at` seconds after the start of a `WINDOW` ending now. +async fn sample(db: &Pool, id: &str, value: serde_json::Value, at: f64) { + sqlx::query( + "INSERT INTO metrics (id, value, created_at) VALUES ($1, $2, now() - make_interval(secs => $3))", + ) + .bind(id) + .bind(value) + .bind(WINDOW - at) + .execute(db) + .await + .expect("failed to store a metric sample"); +} + +/// The database hands the renderer the last sample before the window, which sets the left edge, +/// and for each slot its peak and its latest value, which the line continues from. +#[sqlx::test(migrations = "../migrations")] +async fn a_series_starts_from_the_sample_before_the_window_and_keeps_each_slot_peak( + db: Pool, +) { + // Before the window: 1, then 2, which is what is in force at the left edge. + sample(&db, "queue_count_t", json!(1), -120.0).await; + sample(&db, "queue_count_t", json!(2), -60.0).await; + // Three samples inside one 30s slot: the line rises to their peak, then drops to the last. + sample(&db, "queue_count_t", json!(5), 605.0).await; + sample(&db, "queue_count_t", json!(9), 612.0).await; + sample(&db, "queue_count_t", json!(4), 620.0).await; + // Drained before the window: nothing left to draw. + sample(&db, "queue_count_gone", json!(3), -300.0).await; + sample(&db, "queue_count_gone", json!(0), -200.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + + assert_eq!( + series.tags.len(), + 1, + "a tag drained before the window is left out" + ); + let tag = &series.tags[0]; + assert_eq!(tag.tag, "t"); + assert!(tag.delay.is_empty()); + + let stale = 620.0 + QUEUE_METRIC_STALE_SECS; + let expected = [ + (0.0, 2.0), + (605.0, 2.0), + (605.0, 9.0), + (620.0, 9.0), + (620.0, 4.0), + (stale, 4.0), + (stale, 0.0), + (WINDOW, 0.0), + ]; + assert_eq!(tag.count.len(), expected.len(), "vertices: {:?}", tag.count); + for ((ms, value), (at, expected_value)) in tag.count.iter().zip(expected) { + let secs = (*ms - series.from) as f64 / 1000.0; + // Samples are stored a few milliseconds before the window is read. + assert!( + (secs - at).abs() < 2.0 && *value == expected_value, + "expected ({at}, {expected_value}), got ({secs}, {value}) in {:?}", + tag.count + ); + } +} + +/// A delay stored as its head's wait start is drawn as that wait, growing a second per second, +/// right up to the zero that closes it. +#[sqlx::test(migrations = "../migrations")] +async fn a_climbing_delay_is_drawn_as_the_wait_of_its_head(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // The head started waiting 30s before the window; heartbeats restate it until the drain. + let head = json!({ "since": now - WINDOW - 30.0 }); + for at in [60.0, 360.0, 660.0] { + sample(&db, "queue_delay_t", head.clone(), at).await; + } + sample(&db, "queue_delay_t", json!(0), 900.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let points = series.tags[0] + .delay + .iter() + .map(|(ms, value)| ((*ms - series.from) as f64 / 1000.0, *value)) + .collect::>(); + + let climb = points + .iter() + .filter(|(_, value)| *value > 0.0) + .collect::>(); + assert!( + climb.len() > 4, + "the climb has vertices along the way: {points:?}" + ); + for (at, value) in &climb { + assert!( + (value - (at + 30.0)).abs() < 2.0, + "off the climb at {at}: {points:?}" + ); + } + let (first, _) = climb[0]; + let (top, _) = climb[climb.len() - 1]; + assert!( + (first - 60.0).abs() < 2.0 && (top - 900.0).abs() < 2.0, + "{points:?}" + ); +} + +/// A climb that drains inside its slot keeps its top, which no stored value holds: it is reached +/// at the next sample. +#[sqlx::test(migrations = "../migrations")] +async fn a_climb_that_drains_inside_its_slot_keeps_its_top(db: Pool) { + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + // All in the 30s slot starting at 600: held at 5s, then climbing from a head queued at 597, + // which is still there when the tag drains at 627, 30s into its wait. + sample(&db, "queue_delay_t", json!(5), 602.0).await; + sample(&db, "queue_delay_t", json!({ "since": now - WINDOW + 597.0 }), 610.0).await; + sample(&db, "queue_delay_t", json!(0), 627.0).await; + + let series = read_queue_metrics_series(&db, WINDOW).await.unwrap(); + let top = series.tags[0] + .delay + .iter() + .map(|(_, value)| *value) + .fold(0.0, f64::max); + assert!((top - 30.0).abs() < 2.0, "{:?}", series.tags[0].delay); +} diff --git a/backend/windmill-common/tests/queue_stats.rs b/backend/windmill-common/tests/queue_stats.rs new file mode 100644 index 0000000000..419ad98d56 --- /dev/null +++ b/backend/windmill-common/tests/queue_stats.rs @@ -0,0 +1,69 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::queue::get_queue_stats; + +const WORKSPACE: &str = "test-workspace"; + +async fn queue_job( + db: &Pool, + tag: &str, + priority: Option, + waited_secs: f64, + running: bool, +) { + sqlx::query( + "WITH job AS ( + INSERT INTO v2_job (id, workspace_id, tag) VALUES (gen_random_uuid(), $1, $2) + RETURNING id + ) + INSERT INTO v2_job_queue (id, workspace_id, tag, priority, running, scheduled_for) + SELECT id, $1, $2, $3, $4, now() - make_interval(secs => $5) FROM job", + ) + .bind(WORKSPACE) + .bind(tag) + .bind(priority) + .bind(running) + .bind(waited_secs) + .execute(db) + .await + .expect("failed to queue job"); +} + +/// The delay reported for a tag is that of the job the worker pull takes first, ordered +/// `priority DESC NULLS LAST, scheduled_for`, not simply the oldest one waiting. Running jobs +/// and jobs less than 3 seconds past due are not part of the backlog at all. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn queue_stats_report_the_delay_of_the_job_pulled_next(db: Pool) { + // The oldest job has no priority, so every prioritized job runs before it. + queue_job(&db, "mixed", None, 900.0, false).await; + queue_job(&db, "mixed", Some(1), 600.0, false).await; + queue_job(&db, "mixed", Some(5), 300.0, false).await; + queue_job(&db, "mixed", Some(5), 100.0, false).await; + // Highest priority, but not backlog: already running, or not yet 3 seconds past due. + queue_job(&db, "mixed", Some(9), 1200.0, true).await; + queue_job(&db, "mixed", Some(9), 1.0, false).await; + queue_job(&db, "unprioritized", None, 500.0, false).await; + queue_job(&db, "unprioritized", None, 50.0, false).await; + + let stats = get_queue_stats(&db).await.unwrap(); + let now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::double precision") + .fetch_one(&db) + .await + .unwrap(); + + let mixed = &stats["mixed"]; + assert_eq!(mixed.count, 4); + assert!( + (mixed.delay - 300.0).abs() < 5.0, + "expected the oldest job of the highest priority, got a delay of {}", + mixed.delay + ); + // The same job's wait start, which the delay is measured from. + assert!((mixed.head_since + mixed.delay - now).abs() < 5.0); + let unprioritized = &stats["unprioritized"]; + assert_eq!(unprioritized.count, 2); + assert!( + (unprioritized.delay - 500.0).abs() < 5.0, + "expected the oldest job, got a delay of {}", + unprioritized.delay + ); +} diff --git a/backend/windmill-common/tests/schema_replica_identity.rs b/backend/windmill-common/tests/schema_replica_identity.rs new file mode 100644 index 0000000000..fb14a5c78e --- /dev/null +++ b/backend/windmill-common/tests/schema_replica_identity.rs @@ -0,0 +1,46 @@ +//! Every table must be replicable. +//! +//! PostgreSQL refuses UPDATE and DELETE on a table that has neither a PRIMARY KEY +//! nor an explicit REPLICA IDENTITY once the database is published to a logical +//! replication slot. That is what a low-downtime major-version upgrade runs on +//! (RDS and Aurora Blue/Green, pglogical) and what every CDC pipeline reads, so a +//! single keyless table blocks the upgrade outright. This runs against a freshly +//! migrated database and fails on the migration that introduces one. + +use sqlx::{Pool, Postgres}; + +/// Partitioned parents are checked alongside ordinary tables: a parent without a +/// key hands the same defect to every partition created under it later. +#[sqlx::test(migrations = "../migrations")] +async fn every_table_is_replicable(db: Pool) -> anyhow::Result<()> { + let offenders: Vec = sqlx::query_scalar( + "SELECT n.nspname || '.' || c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND NOT ( + -- FULL and USING INDEX replicate on their own. + c.relreplident IN ('f', 'i') + -- DEFAULT resolves to the primary key, so it needs one to exist. + -- NOTHING never replicates, primary key or not. + OR (c.relreplident = 'd' AND EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary + )) + ) + ORDER BY 1", + ) + .fetch_all(&db) + .await?; + + assert!( + offenders.is_empty(), + "logical replication will reject UPDATE and DELETE on these tables, because \ + they carry no replica identity it can use: {}. \ + Give each one a primary key -- a natural composite key where every column \ + is NOT NULL, otherwise a surrogate `BIGINT GENERATED ALWAYS AS IDENTITY`.", + offenders.join(", ") + ); + + Ok(()) +} diff --git a/backend/windmill-common/tests/user_drafts_rename.rs b/backend/windmill-common/tests/user_drafts_rename.rs new file mode 100644 index 0000000000..35d2b53858 --- /dev/null +++ b/backend/windmill-common/tests/user_drafts_rename.rs @@ -0,0 +1,28 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::user_drafts::rename_drafts_of_email; + +/// A rename onto the same address has to be a no-op: the helper clears a draft the destination +/// already holds at the same item, and every row would be its own destination. SCIM PATCH sends +/// `userName` unconditionally, so an IdP re-sending an unchanged one reaches this. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn renaming_onto_the_same_address_keeps_the_drafts(db: Pool) { + sqlx::query( + "INSERT INTO draft(workspace_id, path, typ, value, email) \ + VALUES ('test-workspace', 'u/test-user/s', 'script', '{}'::json, 'test@windmill.dev')", + ) + .execute(&db) + .await + .expect("failed to seed draft"); + + let mut conn = db.acquire().await.unwrap(); + rename_drafts_of_email(&mut conn, "test@windmill.dev", "test@windmill.dev") + .await + .unwrap(); + + let kept: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM draft WHERE email = 'test@windmill.dev'") + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(kept, 1); +} diff --git a/backend/windmill-common/tests/usr_accepts_email.rs b/backend/windmill-common/tests/usr_accepts_email.rs new file mode 100644 index 0000000000..624d8812b4 --- /dev/null +++ b/backend/windmill-common/tests/usr_accepts_email.rs @@ -0,0 +1,59 @@ +//! `usr_accepts_email` predicts whether `usr` will store an address by evaluating +//! `PROPER_EMAIL_PATTERN` in the database. It only stays right while that text matches the +//! `proper_email` constraint and the width matches the column: each sample below must be +//! stored by `usr` exactly when the check accepts it, and everything `VALID_EMAIL` accepts +//! within the width must be stored too. + +use sqlx::{Pool, Postgres}; +use windmill_common::users::{usr_accepts_email, EMAIL_COLUMN_MAX_LEN, VALID_EMAIL}; + +#[sqlx::test(migrations = "../migrations")] +async fn usr_accepts_email_agrees_with_the_constraint(db: Pool) -> anyhow::Result<()> { + let domain = "@example.com"; + let widest = format!( + "{}{domain}", + "a".repeat(EMAIL_COLUMN_MAX_LEN - domain.len()) + ); + let too_wide = format!("a{widest}"); + for email in [ + "alice@example.com", + "Alice@Example.COM", + "alice.bob+tag@sub.example.co.uk", + "\"quoted\"@example.com", + "\"quoted local\"@example.com", + "alice@[192.168.0.1]", + widest.as_str(), + too_wide.as_str(), + "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed", + // Unicode case folding would map the long s and the Kelvin sign into `[a-z]`. + "u\u{17f}er@example.com", + "alice@example\u{212a}.com", + "alice", + "alice@example", + "alice@@example.com", + "alice @example.com", + "alice@example.com\nbob@example.com", + "", + ] { + let mut tx = db.begin().await?; + let stored = sqlx::query( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator) + VALUES ('admins', 'probe', $1, false, false)", + ) + .bind(email) + .execute(&mut *tx) + .await + .is_ok(); + tx.rollback().await?; + + assert_eq!( + stored, + usr_accepts_email(&db, email).await?, + "{email:?}: `usr` and usr_accepts_email disagree" + ); + if VALID_EMAIL.is_match(email) && email.len() <= EMAIL_COLUMN_MAX_LEN { + assert!(stored, "{email:?}: VALID_EMAIL accepts what `usr` rejects"); + } + } + Ok(()) +} diff --git a/backend/windmill-dep-map/Cargo.toml b/backend/windmill-dep-map/Cargo.toml index 28b8552e8e..a93376d167 100644 --- a/backend/windmill-dep-map/Cargo.toml +++ b/backend/windmill-dep-map/Cargo.toml @@ -26,4 +26,5 @@ tracing.workspace = true lazy_static.workspace = true chrono.workspace = true itertools.workspace = true +futures.workspace = true uuid.workspace = true diff --git a/backend/windmill-dep-map/src/ci_tests.rs b/backend/windmill-dep-map/src/ci_tests.rs index 79f1b7e30b..8f56111b35 100644 --- a/backend/windmill-dep-map/src/ci_tests.rs +++ b/backend/windmill-dep-map/src/ci_tests.rs @@ -19,3 +19,13 @@ pub async fn trigger_ci_tests_for_item( ) -> error::Result> { Ok(vec![]) } + +#[cfg(not(feature = "private"))] +pub async fn trigger_all_ci_tests( + _db: &sqlx::Pool, + _w_id: &str, + _email: &str, + _username: &str, +) -> error::Result> { + Ok(vec![]) +} diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index ee6a3c3fd4..aab50795a9 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -1,6 +1,7 @@ pub mod ci_tests; #[cfg(feature = "private")] pub mod ci_tests_ee; +pub mod lock_hash; pub mod scoped_dependency_map; pub mod trigger_dependents; pub mod workspace_dependencies; @@ -127,6 +128,40 @@ pub fn extract_referenced_paths( } } +/// Re-records which paths `script_path` imports and what each one's lock hashes to right now. +/// That snapshot is what a later relock-skip check of this importer compares against, so it +/// has to move whenever the imports may have, whether or not the importer's own lock did. +/// +/// Writes for any path in `w_id` and checks nothing: callers are responsible for having +/// established access to that workspace and script, as a dependency job's push already has. +pub async fn refresh_dependency_map( + db: &sqlx::Pool, + w_id: &str, + script_path: &str, + parent_path: &Option, + code: &str, + script_lang: &Option, +) -> error::Result<()> { + use scoped_dependency_map::ScopedDependencyMap; + + let mut tx = db.begin().await?; + let mut dependency_map = + ScopedDependencyMap::fetch_maybe_rearranged(w_id, script_path, "script", parent_path, db) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + + dependency_map.dissolve(tx).await.commit().await?; + Ok(()) +} + pub async fn process_relative_imports( db: &sqlx::Pool, _job_id: Option, @@ -144,29 +179,7 @@ pub async fn process_relative_imports( use scoped_dependency_map::ScopedDependencyMap; use trigger_dependents::trigger_dependents_to_recompute_dependencies; - // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled - { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, - ) - .await?; - - tx = dependency_map - .patch( - extract_referenced_paths(&code, script_path, *script_lang), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - - dependency_map.dissolve(tx).await.commit().await?; - } + refresh_dependency_map(db, w_id, script_path, &parent_path, code, script_lang).await?; { let mut already_visited = args diff --git a/backend/windmill-dep-map/src/lock_hash.rs b/backend/windmill-dep-map/src/lock_hash.rs new file mode 100644 index 0000000000..50bd18a8ee --- /dev/null +++ b/backend/windmill-dep-map/src/lock_hash.rs @@ -0,0 +1,79 @@ +use std::collections::HashMap; + +use futures::TryStreamExt; +use sqlx::{Postgres, Transaction}; +use windmill_common::error::Result; +use windmill_common::scripts::hash_script; + +/// Records what the lock now at each path hashes to, which is one half of the comparison a relock +/// skip makes against what each importer resolved against. +/// +/// Writes any path in `w_id` and checks nothing: callers are responsible for having established +/// the caller's access to that workspace. A path repeated in `entries` keeps its last hash. +/// +/// Callers that write the lock itself in the same statement fold the upsert into that statement +/// instead; this is for the ones with nothing to fold it into. +pub async fn record_lock_hashes( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + entries: &[(String, i64)], +) -> Result<()> { + // Postgres rejects a whole statement that resolves a conflict on one key twice, so a path + // given more than once keeps its last hash, as it would if the two were written in order. + let mut deduped: HashMap<&str, i64> = HashMap::with_capacity(entries.len()); + for (path, hash) in entries { + deduped.insert(path.as_str(), *hash); + } + if deduped.is_empty() { + return Ok(()); + } + let (paths, hashes): (Vec, Vec) = deduped + .into_iter() + .map(|(path, hash)| (path.to_string(), hash)) + .unzip(); + // Recording a hash a path already has would still cut a row version, and the no-op push this + // is reached from is the mode a git-sync of an unchanged workspace runs in. + sqlx::query!( + "INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + SELECT $1, * FROM UNNEST($2::text[], $3::bigint[]) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash + WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + w_id, + &paths[..], + &hashes[..] + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Records the hash of every live lock in `w_id`, for a workspace whose scripts arrived without +/// going through a deploy — a clone, which copies their locks verbatim and so would otherwise hold +/// none of the hashes describing them. +/// +/// Carries the same caller obligation as [`record_lock_hashes`]. +/// +/// `script.lock` is unbounded and a workspace holds one per script, so the rows are streamed and +/// each lock is hashed and dropped before the next arrives; only the hashes accumulate. +pub async fn record_lock_hashes_for_workspace( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, +) -> Result<()> { + let mut entries: Vec<(String, i64)> = Vec::new(); + { + let mut rows = sqlx::query!( + "SELECT DISTINCT ON (path) path, lock FROM script + WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL + ORDER BY path, created_at DESC", + w_id + ) + .fetch(&mut **tx); + + while let Some(row) = rows.try_next().await? { + if let Some(lock) = row.lock { + entries.push((row.path, hash_script(&lock))); + } + } + } + record_lock_hashes(tx, w_id, &entries).await +} diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index 148746dcca..2b691ee48e 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,8 +9,8 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] -private = ["windmill-common/private"] -enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +private = ["windmill-common/private", "windmill-dep-map/private", "windmill-audit/private"] +enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise", "windmill-audit/enterprise"] all_sqlx_features = ["enterprise"] default = [] @@ -22,5 +22,7 @@ serde_json.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true +windmill-dep-map.workspace = true +windmill-audit.workspace = true regex = "1.10.3" tokio = { workspace = true, features = ["full"] } \ No newline at end of file diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 2bfc7d2f77..d65e230988 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -14,12 +14,21 @@ pub mod git_sync_oss; #[cfg(feature = "private")] pub use git_sync_ee::{ - enqueue_git_pull_dry_run, enqueue_git_pull_job, handle_deployment_metadata, - handle_deployment_metadata_batch, handle_fork_branch_creation, persist_auto_pull_state, - reconcile_and_enqueue_pull, reconcile_fork_branch_pull, record_auto_pull_failure, + clear_auto_pull_failure, enqueue_git_pull_dry_run, enqueue_git_pull_job, + handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, + persist_auto_pull_state, reconcile_and_enqueue_pull, reconcile_fork_branch_pull, + record_auto_pull_failure, record_synced_head, sweep_ci_test_checks, tally_deployed_object_changes, }; +// The CI-test check exists only on enterprise builds; `private` alone (the CE image) +// compiles git_sync_ee without them. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub use git_sync_ee::{ + ensure_ci_test_check_for_pr, evaluate_and_conclude_ci_test_checks, + post_ci_test_check_not_applicable, resolve_pr_head_workspace, +}; + #[cfg(not(feature = "private"))] pub use git_sync_oss::{ handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index acf90ab57a..09b2bbe5b1 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -10,7 +10,13 @@ path = "src/lib.rs" [features] default = [] -parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] +parquet = [ + "windmill-common/parquet", + "windmill-object-store/parquet", + "dep:datafusion", + "dep:object_store", + "dep:url", +] private = ["windmill-common/private"] enterprise = ["windmill-common/enterprise", "windmill-object-store/enterprise"] @@ -33,3 +39,6 @@ astral-tokio-tar.workspace = true lazy_static.workspace = true const_format.workspace = true flume.workspace = true +datafusion = { workspace = true, optional = true } +object_store = { workspace = true, optional = true } +url = { workspace = true, optional = true } diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 6c13b551d1..f107ea911d 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -7,3 +7,5 @@ pub mod indexer_oss; #[cfg(feature = "private")] pub mod service_logs_ee; pub mod service_logs_oss; +#[cfg(all(feature = "private", feature = "parquet"))] +pub mod service_logs_store_ee; diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs index 3f88f7e781..d49fa468ae 100644 --- a/backend/windmill-mcp/src/common/schema.rs +++ b/backend/windmill-mcp/src/common/schema.rs @@ -101,7 +101,7 @@ fn apply_resource_enrichment( let resources_count = resource_cache.len(); let description = match resource_type { Some(rt) => format!( - "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", + "This is a resource named `{}` with the following description: `{}`.\nPass it as the bare string `$res:` — the whole value of this argument, never an object wrapper like {{\"$res\": \"\"}} and never a plain path.\n{}", rt.name, rt.description.as_deref().unwrap_or("No description"), if resources_count == 0 { @@ -138,7 +138,7 @@ fn apply_resource_enrichment( ) }) .collect::>() - .join("\\n"); + .join("\n"); let prior_description = prop_map .get("description") .and_then(Value::as_str) @@ -147,7 +147,7 @@ fn apply_resource_enrichment( prop_map.insert( "description".to_string(), Value::String(format!( - "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", + "{}\nHere are the available resources, one per line as `title: $res:path`. The title is only a label; pass the `$res:path` part verbatim as this argument's value:\n{}", prior_description, resources_description )), ); @@ -804,6 +804,10 @@ mod tests { let desc = node["description"].as_str().unwrap(); assert!(desc.contains("c_aws_account")); assert!(desc.contains("$res:f/platform/aws_dev")); + // MCP clients render this description verbatim, so the separators must be + // real newlines rather than the two-character escape. + assert!(desc.contains('\n')); + assert!(!desc.contains("\\n")); } #[test] diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index e42c0dacfc..7ca1eb08a1 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -16,6 +16,14 @@ use crate::server::endpoints::EndpointTool; /// Result type for backend operations using rmcp's ErrorData directly pub type BackendResult = Result; +/// What the backend needs about the HTTP request a tool call arrived on, in order +/// to hand a runnable the headers of the call that triggered it. +pub struct McpRequest<'a> { + pub headers: &'a http::HeaderMap, + /// The MCP tool name the caller invoked, reported to preprocessors. + pub tool_name: &'a str, +} + /// How a script/flow listing is narrowed by path at the SQL layer, *before* the /// `ITEMS_FETCH_MAX_LIMIT` cap applies. /// @@ -157,6 +165,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Run a flow and wait for result @@ -166,6 +175,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Call an endpoint tool (generated API endpoint) diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index b6fb7a5b0a..b97e374e98 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -12,7 +12,7 @@ pub mod tools; // Re-export main types pub use crate::common::types::{McpToken, MultiWorkspaceMcp, WorkspaceInfo}; -pub use backend::{BackendResult, McpAuth, McpBackend, PathFilter}; +pub use backend::{BackendResult, McpAuth, McpBackend, McpRequest, PathFilter}; pub use endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only, list_workspaces_tool, non_empty_body_fields, EndpointTool, diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index f407504c4b..e27eaa8470 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -9,8 +9,10 @@ use crate::common::transform::{ extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, }; -use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId}; -use crate::server::backend::{McpAuth, McpBackend, PathFilter}; +use crate::common::types::{ + McpToken, MultiWorkspaceMcp, ResourceInfo, SchemaType, ToolableItem, WorkspaceId, +}; +use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter}; use crate::server::endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; @@ -101,16 +103,24 @@ enum McpMode { Multi(String), } +/// Everything a request carries besides its MCP payload. +struct McpContext { + auth: A, + mode: McpMode, + headers: http::HeaderMap, +} + impl Runner { /// Create a new Runner with the given backend pub fn new(backend: B) -> Self { Self { backend: Arc::new(backend) } } - /// Extract authentication and the workspace mode from request context + /// Extract authentication, the workspace mode and the HTTP request itself + /// from the request context fn extract_context( context: &RequestContext, - ) -> Result<(B::Auth, McpMode), ErrorData> { + ) -> Result, ErrorData> { let http_parts = context.extensions.get::().ok_or_else(|| { tracing::error!("http::request::Parts not found"); ErrorData::internal_error("http::request::Parts not found", None) @@ -148,7 +158,7 @@ impl Runner { McpMode::Single(workspace_id) }; - Ok((auth.clone(), mode)) + Ok(McpContext { auth: auth.clone(), mode, headers: http_parts.headers.clone() }) } } @@ -391,6 +401,18 @@ fn authorize_endpoint_call( Ok(()) } +/// Map the model's argument keys back to the runnable's original parameter names. +fn transform_call_args(args: Value, item_schema: &Option) -> Value { + let Value::Object(map) = args else { + return args; + }; + let mut args_hash = HashMap::new(); + for (k, v) in map { + args_hash.insert(reverse_transform_key(&k, item_schema), v); + } + Value::Object(args_hash.into_iter().collect()) +} + fn find_matching_path(candidates: Vec, request_name: &str) -> Option { candidates .into_iter() @@ -427,7 +449,7 @@ impl ServerHandler for Runner { _request: Option, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, .. } = Self::extract_context(&context)?; // Parse MCP scopes to determine what to expose let scopes = auth.scopes().unwrap_or(&[]); @@ -455,7 +477,7 @@ impl ServerHandler for Runner { request: CallToolRequestParams, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, headers } = Self::extract_context(&context)?; // Parse MCP scopes for authorization let scopes = auth.scopes().unwrap_or(&[]); @@ -464,6 +486,7 @@ impl ServerHandler for Runner { let read_only = auth.read_only(); let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); + let mcp_request = McpRequest { headers: &headers, tool_name: request.name.as_ref() }; // Every tool here runs to completion in one round trip: none of them ask the // client for input, so the MRTR variants of `CallToolResponse` are never built. @@ -474,14 +497,22 @@ impl ServerHandler for Runner { &workspace_id, &scope_config, read_only, - request.name, + request.name.clone(), args, + &mcp_request, ) .await } McpMode::Multi(token) => { - self.call_tool_multi(&auth, &token, &scope_config, read_only, request.name, args) - .await + self.call_tool_multi( + &auth, + &token, + &scope_config, + read_only, + request.name.clone(), + args, + ) + .await } }?; Ok(result.into()) @@ -665,6 +696,7 @@ impl Runner { read_only: bool, name: std::borrow::Cow<'static, str>, args: Value, + request: &McpRequest<'_>, ) -> Result { // Check if this is an endpoint tool let endpoint_tools = self.backend.all_endpoint_tools(); @@ -777,17 +809,7 @@ impl Runner { .map_err(|e| ErrorData::internal_error(e.message, None))? }; - // Transform arguments back to original key names - let transformed_args = if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); - for (k, v) in map { - let original_key = reverse_transform_key(&k, &item_schema); - args_hash.insert(original_key, v); - } - Value::Object(args_hash.into_iter().collect()) - } else { - args - }; + let transformed_args = transform_call_args(args, &item_schema); let script_or_flow_path = if is_hub { format!("hub/{}", path) @@ -798,11 +820,23 @@ impl Runner { // Execute script or flow let result = if tool_type == "script" { self.backend - .run_script(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_script( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await } else { self.backend - .run_flow(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_flow( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await }; diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index cc6c61be81..27a50ba07a 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -1,8 +1,8 @@ use crate::{ classify_read_failure, decrypt_oauth_data, delete_native_trigger, delete_token_by_hash, get_native_trigger, list_native_triggers, lock::TriggerLock, map_external_error, - map_external_error_with, rotate_webhook_token, store_native_trigger, - sync::EXTERNAL_TRIGGER_MISSING_ERROR, update_native_trigger_error, + map_external_error_with, rotate_webhook_token, set_native_trigger_enabled, + store_native_trigger, sync::EXTERNAL_TRIGGER_MISSING_ERROR, update_native_trigger_error, update_native_trigger_if_runnable_unchanged, webhook_token_label, webhook_token_scopes, External, ExternalReadFailure, NativeTrigger, NativeTriggerConfig, NativeTriggerData, ServiceName, @@ -239,6 +239,7 @@ async fn create_native_trigger( &config, service_config, data.summary.as_deref(), + data.enabled, ) .await?; @@ -603,6 +604,87 @@ async fn delete_native_trigger_handler( Ok(format!("Native trigger deleted")) } +#[derive(Debug, Deserialize)] +pub struct SetEnabledPayload { + pub enabled: bool, +} + +/// Pause or resume a trigger, without touching its registration on the external service. +/// +/// Leaving the webhook registered is what makes this reversible: services drop or deactivate a +/// subscription that keeps failing, so a paused trigger keeps answering deliveries normally and +/// simply starts no job. +async fn set_native_trigger_enabled_handler( + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) + .await? + .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; + + check_scopes(&authed, || { + format!("native_triggers:write:{}", &existing.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &existing.script_path, + existing.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let updated = set_native_trigger_enabled( + &mut *tx, + &workspace_id, + service_name, + &external_id, + payload.enabled, + ) + .await?; + + // The read above takes no row lock, so a concurrent delete can land in between; reporting + // success then would tell the caller a trigger that is gone had been paused. + if !updated { + return Err(Error::NotFound(format!( + "Native trigger not found: {}", + external_id + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!( + "native_triggers.{}.{}", + service_name, + if payload.enabled { "enable" } else { "disable" } + ), + ActionKind::Update, + &workspace_id, + Some(&external_id), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "Native trigger {}", + if payload.enabled { + "enabled" + } else { + "disabled" + } + )) +} + async fn list_native_triggers_handler( Extension(service_name): Extension, authed: ApiAuthed, @@ -642,6 +724,10 @@ pub fn service_routes(handler: T) -> Router { .route( "/delete/{external_id}", delete(delete_native_trigger_handler::), + ) + .route( + "/setenabled/{external_id}", + post(set_native_trigger_enabled_handler::), ); standard_routes diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4400091ed6..570319ee0e 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -226,6 +226,10 @@ pub struct NativeTrigger { pub created_at: DateTime, pub updated_at: DateTime, pub summary: Option, + /// Whether incoming webhooks for this trigger start a job. Operational state: a create sets + /// its initial value and `setenabled` is its only mutator afterwards, so saving a + /// configuration can never silently re-enable a trigger someone paused. + pub enabled: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -235,12 +239,20 @@ pub struct NativeTriggerConfig { pub webhook_token: String, } +fn default_true() -> bool { + true +} + #[derive(Debug, Serialize, Deserialize)] pub struct NativeTriggerData { pub script_path: String, pub is_flow: bool, pub service_config: C, pub summary: Option, + /// Honoured on create only, so a trigger can be registered already paused in one request. + /// An update ignores it: `setenabled` is the only way to change an existing trigger's state. + #[serde(default = "default_true")] + pub enabled: bool, } #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] @@ -1179,11 +1191,15 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> config: &NativeTriggerConfig, service_config: C, summary: Option<&str>, + enabled: bool, ) -> Result<()> { use windmill_common::auth::hash_token; let webhook_token_hash = hash_token(&config.webhook_token); + // `enabled` is set by the INSERT alone: writing it here rather than in a follow-up statement + // is what keeps a trigger created paused from ever being visible, and therefore runnable, in + // any other state. The conflict branch leaves it untouched for the mirror-image reason. sqlx::query!( r#" INSERT INTO native_trigger ( @@ -1194,9 +1210,10 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> is_flow, webhook_token_hash, service_config, - summary + summary, + enabled ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8 + $1, $2, $3, $4, $5, $6, $7, $8, $9 ) ON CONFLICT (external_id, workspace_id, service_name) DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW() @@ -1209,6 +1226,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> webhook_token_hash, sqlx::types::Json(service_config) as _, summary, + enabled, ) .execute(db) .await?; @@ -1405,7 +1423,8 @@ pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( error, created_at, updated_at, - summary + summary, + enabled FROM native_trigger WHERE @@ -1444,7 +1463,8 @@ pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = P error, created_at, updated_at, - summary + summary, + enabled FROM native_trigger WHERE @@ -1491,7 +1511,8 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> nt.error, nt.created_at, nt.updated_at, - nt.summary + nt.summary, + nt.enabled FROM native_trigger nt WHERE @@ -1555,6 +1576,71 @@ pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Po Ok(()) } +/// Pause or resume a trigger. Returns `false` when there is no such trigger. +/// +/// Callers MUST have verified write access to the trigger's runnable: this writes operational +/// state and performs no authorization of its own. +pub async fn set_native_trigger_enabled<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + enabled: bool, +) -> Result { + // `updated_at` is the row version `record_reregistration` conditions on, so leave it alone: + // pausing a trigger must not make a registration that is mid-flight discard its result. + let updated = sqlx::query!( + r#" + UPDATE native_trigger + SET enabled = $1 + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + enabled, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await? + .rows_affected(); + + Ok(updated > 0) +} + +/// Whether a webhook arriving for this trigger should start a job. +/// +/// A trigger Windmill no longer knows about counts as enabled: the token in the URL is what +/// authorizes the run, and this is a pause switch, not a second authorization check. It reads +/// nothing a caller could not already learn from the trigger it is delivering for, so it needs no +/// authorization of its own — but it also grants none, and must not be used as one. +pub async fn native_trigger_is_enabled<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result { + let enabled = sqlx::query_scalar!( + r#" + SELECT enabled + FROM native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id, + ) + .fetch_optional(db) + .await?; + + Ok(enabled.unwrap_or(true)) +} + pub async fn update_native_trigger_service_config< 'c, E: sqlx::Executor<'c, Database = Postgres>, @@ -1701,6 +1787,11 @@ pub async fn delete_workspace_integration( /// /// `external_id` is optional because during CREATE we don't have it yet /// (it's returned by the external service). During UPDATE, we have it. +/// +/// Every registered URL MUST end up carrying it: it is the only thing a delivery identifies its +/// trigger by, so a service that leaves it out ships a disable switch that silently does nothing. +/// A handler that returns `None` from `service_config_from_create_response` gets this for free — +/// `create_native_trigger` then runs the `update` cycle that re-registers with the assigned id. pub fn generate_webhook_service_url( base_url: &str, w_id: &str, diff --git a/backend/windmill-native-triggers/src/rename.rs b/backend/windmill-native-triggers/src/rename.rs index a7168f5c7e..caae99028c 100644 --- a/backend/windmill-native-triggers/src/rename.rs +++ b/backend/windmill-native-triggers/src/rename.rs @@ -135,6 +135,7 @@ async fn reregister_one( is_flow: trigger.is_flow, service_config, summary: trigger.summary.clone(), + enabled: trigger.enabled, }; // The token is scoped to the runnable path and only its hash is kept, so pointing the webhook diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 2650b68b4d..a8a8e83883 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -88,6 +88,10 @@ pub struct OAuthConfig { #[serde(default = "empty_string")] pub token_url: String, pub userinfo_url: Option, + /// The registry JSON may also carry two frontend-only keys for the connect + /// dialog, deliberately not modelled here: `scope_options`, a scope pick + /// list, and `resource_fields`, the fields of the resource type the dialog + /// asks for once the token is in (Snowflake's database and warehouse). pub scopes: Option>, /// Default scopes for the client-credentials (2-legged) flow. These differ /// from the authorization-code `scopes` for most providers (member/consent diff --git a/backend/windmill-object-store/Cargo.toml b/backend/windmill-object-store/Cargo.toml index 3602007d8c..90af4f7e7c 100644 --- a/backend/windmill-object-store/Cargo.toml +++ b/backend/windmill-object-store/Cargo.toml @@ -58,3 +58,4 @@ aws-credential-types = { workspace = true, optional = true } tempfile.workspace = true tokio = { workspace = true, features = ["rt", "macros"] } object_store.workspace = true +serial_test = "3" diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cc588d485f..edc3ea9149 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -68,8 +68,8 @@ pub mod object_store_reexports { pub use object_store::path::Path; pub use object_store::{ Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult, - ObjectStore, PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, - WriteMultipart, + ObjectMeta, ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult, + Result as ObjectStoreResult, UpdateVersion, WriteMultipart, }; } @@ -118,6 +118,10 @@ pub fn object_store_error_to_error(err: object_store::Error) -> error::Error { pub struct ExpirableObjectStore { pub store: Arc, pub refresh: Option, + /// What locates the store's objects ([`object_store_location`]), for a store built from + /// settings. Kept with the store rather than read off the settings again, so a server + /// whose reload is still pending never names one store by another's location. + pub location: Option, } #[cfg(feature = "parquet")] @@ -155,44 +159,234 @@ impl ObjectStoreRefresh { #[cfg(feature = "parquet")] impl From> for ExpirableObjectStore { fn from(store: Arc) -> Self { - Self { store, refresh: None } + Self { store, refresh: None, location: None } } } #[cfg(feature = "parquet")] lazy_static::lazy_static! { pub static ref OBJECT_STORE_SETTINGS: Arc>> = Arc::new(RwLock::new(None)); + + /// Worker-group override of the store backing the *dependency cache* only: venvs, language + /// bundles and compiled binaries, which a worker both writes and reads back itself. + /// Everything the server also reads — job results, logs, codebases, app assets — stays on + /// [`OBJECT_STORE_SETTINGS`], which a worker-local redirect would make unreachable. + static ref CACHE_OBJECT_STORE_OVERRIDE: Arc>> = Arc::new(RwLock::new(None)); + + /// The config [`CACHE_OBJECT_STORE_OVERRIDE`] was built from, so a rebuild that fails for a + /// config already being served can keep serving it. Locked after the store, never before. + static ref CACHE_OVERRIDE_APPLIED: Arc>> = Arc::new(RwLock::new(None)); + + /// Held across a whole [`reload_cache_object_store_override`], build included, so that the + /// override's flag, store and applied config only ever move together. + static ref CACHE_OVERRIDE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); +} + +/// Whether a worker-group cache override is configured, held apart from the store it built so +/// that a configured-but-broken override reads as "no cache store" instead of silently falling +/// back to the instance bucket the operator redirected away from. +#[cfg(feature = "parquet")] +static CACHE_OBJECT_STORE_OVERRIDDEN: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Bumped by every [`reload_cache_object_store_override`] at entry. Builds are slow and several +/// callers race — a config change, the retry behind a failed one, a license-plan change — and the +/// lock alone would only order them by arrival, so a reload that lost its claim while waiting +/// drops out rather than installing a store the group has already moved off. +#[cfg(feature = "parquet")] +static CACHE_OVERRIDE_GENERATION: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +#[cfg(feature = "parquet")] +async fn resolve_object_store( + settings_lock: &RwLock>, +) -> Option> { + resolve_object_store_with_location(settings_lock) + .await + .map(|(store, _)| store) +} + +#[cfg(feature = "parquet")] +async fn resolve_object_store_with_location( + settings_lock: &RwLock>, +) -> Option<(Arc, Option)> { + let settings = settings_lock.read().await; + let Some(s) = settings.as_ref() else { + return None; + }; + match &s.refresh { + Some(refresh) if refresh.refresh_needed() => { + let refresh = refresh.clone(); + let refreshed_from = s.store.clone(); + drop(settings); + let new_store = refresh.refresh().await?; + let mut settings = settings_lock.write().await; + match settings.as_ref() { + // A reload may have installed a different store while the credentials were + // being minted; that one reflects newer config, so the refresh is stale. + Some(current) if !Arc::ptr_eq(¤t.store, &refreshed_from) => { + Some((current.store.clone(), current.location.clone())) + } + Some(_) => { + let found = (new_store.store.clone(), new_store.location.clone()); + *settings = Some(new_store); + Some(found) + } + // Cleared while refreshing. + None => None, + } + } + _ => Some((s.store.clone(), s.location.clone())), + } } #[cfg(feature = "parquet")] pub async fn get_object_store() -> Option> { - let settings = OBJECT_STORE_SETTINGS.read().await; - if let Some(s) = settings.as_ref() { - match &s.refresh { - Some(refresh) => { - if refresh.refresh_needed() { - let refresh = refresh.clone(); - drop(settings); - let new_store = refresh.refresh().await; - if let Some(new_store) = new_store { - let mut s3_cache_settings = OBJECT_STORE_SETTINGS.write().await; - let arc = new_store.store.clone(); - *s3_cache_settings = Some(new_store); - return Some(arc); - } else { - return None; - } - } else { - return Some(s.store.clone()); - } - } - None => { - return Some(s.store.clone()); - } - } - } else { - return None; + resolve_object_store(&OBJECT_STORE_SETTINGS).await +} + +/// The instance object store with what locates its objects ([`object_store_location`]), +/// read together; the location is `None` for a store installed without settings. +#[cfg(feature = "parquet")] +pub async fn get_object_store_with_location() -> Option<(Arc, Option)> { + resolve_object_store_with_location(&OBJECT_STORE_SETTINGS).await +} + +/// The store the dependency cache reads and writes: the worker group's override when it has one, +/// the instance object store otherwise. Anything the server must also reach goes through +/// [`get_object_store`] instead. +#[cfg(feature = "parquet")] +pub async fn get_cache_object_store() -> Option> { + if CACHE_OBJECT_STORE_OVERRIDDEN.load(std::sync::atomic::Ordering::Relaxed) { + return resolve_object_store(&CACHE_OBJECT_STORE_OVERRIDE).await; } + resolve_object_store(&OBJECT_STORE_SETTINGS).await +} + +/// True when an override is configured but has no usable store. The caller's short retry is the +/// fast path back; this is the backstop, and the full settings reload it rides on is 12h apart by +/// default (`SETTINGS_RELOAD_PERIOD_SECS`), so an outage outlasting the retry keeps the group's +/// dependency cache local until then or until someone edits the group config. +#[cfg(feature = "parquet")] +pub async fn cache_object_store_override_failed() -> bool { + CACHE_OBJECT_STORE_OVERRIDDEN.load(std::sync::atomic::Ordering::Relaxed) + && CACHE_OBJECT_STORE_OVERRIDE.read().await.is_none() +} + +/// Apply the `object_store_cache_config` of this worker's group. `None` (or JSON null) drops the +/// override and returns the worker to the instance object store. +/// +/// Returns [`ObjectStoreReload::Later`] when the store did not build for a reason that may pass — +/// the caller is expected to retry shortly, as `initial_load` does for the instance store. +#[cfg(feature = "parquet")] +pub async fn reload_cache_object_store_override( + db: &windmill_common::DB, + settings: Option, +) -> ObjectStoreReload { + use std::sync::atomic::Ordering; + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + // Claim a generation, then take the lock: every state transition below happens inside one + // critical section, and a caller that lost its claim while waiting drops out rather than + // installing what the group has already moved off. + let generation = CACHE_OVERRIDE_GENERATION.fetch_add(1, Ordering::SeqCst) + 1; + let _transition = CACHE_OVERRIDE_LOCK.lock().await; + if CACHE_OVERRIDE_GENERATION.load(Ordering::SeqCst) != generation { + return ObjectStoreReload::Never; + } + + // DISABLE_S3_STORE turns off the instance object store for this process; a group override + // must not be a way back in. + let store_disabled = std::env::var("DISABLE_S3_STORE") + .ok() + .is_some_and(|x| x == "1" || x == "true"); + + let Some(settings) = settings.filter(|v| !v.is_null() && !store_disabled) else { + if CACHE_OBJECT_STORE_OVERRIDDEN.swap(false, Ordering::Relaxed) { + clear_cache_object_store_override().await; + tracing::info!( + "Worker group object store cache override removed, falling back to the instance object store" + ); + } + return ObjectStoreReload::Never; + }; + + // Enterprise-only, so anything else — Community, including a CE build reaching this through + // the config-as-code API, and Pro — must not get a store, and a plan that stops being + // Enterprise must drop one loaded while it still was. + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + tracing::error!( + "Object store cache override requires an enterprise license, ignoring it for this worker group" + ); + if CACHE_OBJECT_STORE_OVERRIDDEN.swap(false, Ordering::Relaxed) { + clear_cache_object_store_override().await; + } + return ObjectStoreReload::Never; + } + + apply_cache_object_store_override(db, settings).await +} + +/// The half of [`reload_cache_object_store_override`] past the entitlement gate: build the store +/// and commit it. Split out so the commit rules are testable without a license plan. +#[cfg(feature = "parquet")] +async fn apply_cache_object_store_override( + db: &windmill_common::DB, + settings: serde_json::Value, +) -> ObjectStoreReload { + use std::sync::atomic::Ordering; + + // Claim the override before building it: until a store is in place the dependency cache + // must stay local-only rather than reach for the instance bucket. + CACHE_OBJECT_STORE_OVERRIDDEN.store(true, Ordering::Relaxed); + + let (store, reload) = match serde_json::from_value::(settings.clone()) { + Ok(setting) => match build_object_store_from_settings(setting, Some(db)).await { + Ok(store) => (Some(store), ObjectStoreReload::Never), + Err(e) => { + tracing::error!( + "Error building the worker group object store cache override, the dependency cache stays local to this worker until it builds: {e:?}" + ); + (None, ObjectStoreReload::Later) + } + }, + // A malformed config will read the same on every retry. + Err(e) => { + tracing::error!( + "Error parsing the worker group object store cache override, the dependency cache stays local to this worker: {e:?}" + ); + (None, ObjectStoreReload::Never) + } + }; + + let mut current = CACHE_OBJECT_STORE_OVERRIDE.write().await; + match store { + Some(store) => { + *current = Some(store); + *CACHE_OVERRIDE_APPLIED.write().await = Some(settings); + tracing::info!( + "Dependency cache of this worker group now uses its own object store, not the instance one" + ); + } + // A rebuild that failed for the config already being served leaves that store in place: + // the group is entitled to it, and dropping it would take the whole group's cache local + // over a transient error. A *different* config failing must still clear, or the worker + // would keep writing to the bucket the operator redirected it away from. + None if current.is_some() + && CACHE_OVERRIDE_APPLIED.read().await.as_ref() == Some(&settings) => {} + None => { + *current = None; + *CACHE_OVERRIDE_APPLIED.write().await = None; + } + } + reload +} + +/// Drop the override store and the config it was built from, in that lock order. +#[cfg(feature = "parquet")] +async fn clear_cache_object_store_override() { + *CACHE_OBJECT_STORE_OVERRIDE.write().await = None; + *CACHE_OVERRIDE_APPLIED.write().await = None; } #[cfg(feature = "parquet")] @@ -248,20 +442,22 @@ pub async fn reload_object_store_setting(db: &windmill_common::DB) -> ObjectStor tracing::error!("S3 cache is not available for pro plan"); return ObjectStoreReload::Never; } - *s3_cache_settings = build_s3_client_from_settings(S3Settings { - bucket: None, - region: None, - access_key: None, - secret_key: None, - endpoint: None, - store_logs: None, - path_style: None, - allow_http: None, - port: None, - }) + *s3_cache_settings = build_object_store_from_settings( + ObjectSettings::S3(S3Settings { + bucket: None, + region: None, + access_key: None, + secret_key: None, + endpoint: None, + store_logs: None, + path_style: None, + allow_http: None, + port: None, + }), + Some(db), + ) .await .ok() - .map(|x| ExpirableObjectStore::from(x)) } else { *s3_cache_settings = None; } @@ -713,19 +909,49 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes { } } +/// What locates a store's objects: endpoint, port, region and bucket (or account and +/// container, or root), never the credentials, which rotate. Two stores with the same +/// location hold the same objects. +pub fn object_store_location(resource: &ObjectStoreResource) -> String { + match resource { + ObjectStoreResource::S3(s) => format!( + "s3:{}:{}:{}:{}", + s.endpoint, + s.port.unwrap_or_default(), + s.region, + s.bucket + ), + ObjectStoreResource::Azure(a) => format!( + "azure:{}:{}:{}", + a.endpoint.as_deref().unwrap_or_default(), + a.account_name, + a.container_name + ), + ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), + ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), + } +} + #[cfg(feature = "parquet")] pub async fn build_object_store_from_settings( settings: ObjectSettings, init_private_key: Option<&windmill_common::DB>, ) -> error::Result { + let located = + |store: Arc, resource: ObjectStoreResource| ExpirableObjectStore { + store, + refresh: None, + location: Some(object_store_location(&resource)), + }; match settings { - ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings) - .await - .map(|x| ExpirableObjectStore::from(x)), - ObjectSettings::Azure(azure_settings) => { - let azure_blob_resource = azure_settings; - build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) + ObjectSettings::S3(s3_settings) => { + let s3_resource = s3_resource_from_settings(s3_settings); + build_s3_client(&s3_resource) + .await + .map(|x| located(x, ObjectStoreResource::S3(s3_resource))) } + ObjectSettings::Azure(azure_settings) => build_azure_blob_client(&azure_settings) + .map(|x| located(x, ObjectStoreResource::Azure(azure_settings))), ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { let token_generator = crate::job_s3_helpers_oss::TokenGenerator::AsServerInstance(); let res = crate::job_s3_helpers_oss::generate_s3_aws_oidc_resource( @@ -740,17 +966,14 @@ pub async fn build_object_store_from_settings( .map(|x| ExpirableObjectStore { store: x, refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())), + location: Some(object_store_location(&res)), }) } - ObjectSettings::Gcs(gcs_settings) => { - let gcs_resource = gcs_settings; - build_gcs_client(&gcs_resource) - .await - .map(|x| ExpirableObjectStore::from(x)) - } - ObjectSettings::Filesystem(fs) => { - build_filesystem_client(&fs.root_path).map(|x| ExpirableObjectStore::from(x)) - } + ObjectSettings::Gcs(gcs_settings) => build_gcs_client(&gcs_settings) + .await + .map(|x| located(x, ObjectStoreResource::Gcs(gcs_settings))), + ObjectSettings::Filesystem(fs) => build_filesystem_client(&fs.root_path) + .map(|x| located(x, ObjectStoreResource::Filesystem(fs))), } } @@ -763,14 +986,14 @@ fn none_if_empty(s: Option) -> Option { } } +/// The S3 resource instance settings resolve to, the environment filling in what they +/// leave out. #[cfg(feature = "parquet")] -pub async fn build_s3_client_from_settings( - settings: S3Settings, -) -> error::Result> { +fn s3_resource_from_settings(settings: S3Settings) -> S3Resource { let region = none_if_empty(settings.region) .unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string())); - let s3_resource = S3Resource { + S3Resource { endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| { std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com")) }), @@ -785,9 +1008,7 @@ pub async fn build_s3_client_from_settings( port: settings.port, token: None, expiration: None, - }; - - build_s3_client(&s3_resource).await + } } // Resolving the default chain goes over the network (ECS/IMDS) on instances relying on an @@ -2361,10 +2582,130 @@ mod tests { .contains("Error building filesystem object store")); } + /// A worker group override that is configured but has no usable store must leave the + /// dependency cache with no object store at all. Falling back to the instance one would + /// write the group's cache into the bucket the operator redirected it away from. + // Serialized with the other test that swaps OBJECT_STORE_SETTINGS: the store is + // process-global and CI runs this binary with --test-threads=10. + #[cfg(feature = "parquet")] + #[tokio::test] + #[serial_test::serial(object_store_settings)] + async fn test_get_cache_object_store_override() { + use object_store::{path::Path, ObjectStore, PutPayload}; + use std::sync::atomic::Ordering; + + async fn marker_of(store: &Arc) -> String { + let bytes = store.get(&Path::from("marker")).await.unwrap(); + String::from_utf8(bytes.bytes().await.unwrap().to_vec()).unwrap() + } + + let instance_dir = tempfile::tempdir().unwrap(); + let instance = build_filesystem_client(instance_dir.path().to_str().unwrap()).unwrap(); + instance + .put(&Path::from("marker"), PutPayload::from("instance")) + .await + .unwrap(); + + let group_dir = tempfile::tempdir().unwrap(); + let group = build_filesystem_client(group_dir.path().to_str().unwrap()).unwrap(); + group + .put(&Path::from("marker"), PutPayload::from("group")) + .await + .unwrap(); + + *OBJECT_STORE_SETTINGS.write().await = Some(ExpirableObjectStore::from(instance)); + + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "instance"); + assert!(!cache_object_store_override_failed().await); + + CACHE_OBJECT_STORE_OVERRIDDEN.store(true, Ordering::Relaxed); + *CACHE_OBJECT_STORE_OVERRIDE.write().await = Some(ExpirableObjectStore::from(group)); + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "group"); + + // Configured but unbuilt, as a failed reload leaves it: no store at all, rather than the + // instance bucket the operator redirected the group away from. + *CACHE_OBJECT_STORE_OVERRIDE.write().await = None; + assert!(get_cache_object_store().await.is_none()); + assert!(cache_object_store_override_failed().await); + + // The teardown branch returns before the pool is used, so a lazy one is enough. + let db = sqlx::postgres::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); + reload_cache_object_store_override(&db, None).await; + let store = get_cache_object_store().await.unwrap(); + assert_eq!(marker_of(&store).await, "instance"); + assert!(!cache_object_store_override_failed().await); + + *OBJECT_STORE_SETTINGS.write().await = None; + } + + /// A rebuild is triggered by any edit to the group config, not only by editing the store, so + /// a build that fails for the config already installed must leave it alone — otherwise a + /// renamed worker tag plus one flaky token mint takes the whole group's cache local. A + /// *different* config failing still has to clear it. + #[cfg(feature = "parquet")] + #[tokio::test] + #[serial_test::serial(object_store_settings)] + async fn test_failed_rebuild_keeps_the_store_serving_the_same_config() { + let db = sqlx::postgres::PgPool::connect_lazy("postgres://localhost/unused").unwrap(); + let dir = tempfile::tempdir().unwrap(); + let settings = serde_json::json!({ + "type": "Filesystem", "root_path": dir.path().to_str().unwrap() + }); + + apply_cache_object_store_override(&db, settings.clone()).await; + assert!(get_cache_object_store().await.is_some()); + + // Same config, now unbuildable: the store it already produced stays. + dir.close().unwrap(); + apply_cache_object_store_override(&db, settings).await; + assert!(get_cache_object_store().await.is_some()); + + // A different config that will not build must not leave the old bucket in place. + let moved = serde_json::json!({ "type": "Filesystem", "root_path": "/proc/nonexistent" }); + apply_cache_object_store_override(&db, moved).await; + assert!(get_cache_object_store().await.is_none()); + assert!(cache_object_store_override_failed().await); + + reload_cache_object_store_override(&db, None).await; + } + + /// A store built from settings is located by where its objects are, not by how the + /// client describes itself: an S3 client prints only its bucket, so the same bucket name + /// on another endpoint would otherwise pass for the same store. + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_settings_store_location_tells_endpoints_apart() { + let s3 = |endpoint: &str| { + ObjectSettings::S3(S3Settings { + bucket: Some("windmill".to_string()), + region: Some("us-east-1".to_string()), + access_key: Some("key".to_string()), + secret_key: Some("secret".to_string()), + endpoint: Some(endpoint.to_string()), + allow_http: Some(true), + path_style: Some(true), + store_logs: None, + port: None, + }) + }; + let a = build_object_store_from_settings(s3("minio.internal:9000"), None) + .await + .unwrap(); + let b = build_object_store_from_settings(s3("s3.us-east-1.amazonaws.com"), None) + .await + .unwrap(); + assert_eq!(a.store.to_string(), b.store.to_string()); + assert!(a.location.is_some()); + assert_ne!(a.location, b.location); + } + // --- get_logs_from_store test --- #[cfg(feature = "parquet")] #[tokio::test] + #[serial_test::serial(object_store_settings)] async fn test_get_logs_from_store_with_filesystem() { use futures::StreamExt; use object_store::{path::Path, ObjectStore, PutPayload}; diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index b9bd6fd28e..6dc5d8ba42 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] private = [] enterprise = ["windmill-common/enterprise"] -cloud = [] +cloud = ["windmill-common/cloud"] benchmark = ["windmill-common/benchmark"] failpoints = [] prometheus = ["dep:prometheus"] diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index a360ae1461..30ff41ad7c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -657,6 +657,16 @@ pub struct ResultMetadata { pub wm_failure: Option, } +/// Parses a marker struct out of a job result, which only an object can carry. +/// A derived `Deserialize` also accepts an array, filling fields by position, so +/// without the check a result like `[[], "boom"]` reads as `wm_failure: "boom"`. +pub fn parse_result_object(result: &str) -> Option { + if !result.trim_start().starts_with('{') { + return None; + } + serde_json::from_str(result).ok() +} + /// Sentinel `error.name` we inject into a result when retagging a successful /// run as a failure due to `wm_failure`. Used downstream to detect that /// the result is already in the standard `{ error: { name, message }, ... }` @@ -677,8 +687,7 @@ pub fn is_pre_shaped_wm_failure_result(result: &str) -> bool { struct NameOnly { name: String, } - serde_json::from_str::(result) - .ok() + parse_result_object::(result) .and_then(|m| m.error) .map(|e| e.name == MANUAL_FAILURE_ERROR_NAME) .unwrap_or(false) @@ -724,7 +733,7 @@ impl ValidableJson for Box { } fn result_metadata(&self) -> ResultMetadata { - serde_json::from_str::(self.get()).unwrap_or_default() + parse_result_object::(self.get()).unwrap_or_default() } fn size(&self) -> usize { @@ -777,6 +786,10 @@ impl ValidableJson for serde_json::Value { } fn result_metadata(&self) -> ResultMetadata { + // An array would decode positionally, see `parse_result_object`. + if !self.is_object() { + return ResultMetadata::default(); + } serde_json::from_value::(self.clone()).unwrap_or_default() } @@ -986,7 +999,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64, Option), Error> { +) -> Result<(Uuid, i64), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1020,7 +1033,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1055,9 +1068,13 @@ pub async fn add_completed_job( .sleep(tokio::time::sleep) .await?; + if wac_parent_ready { + windmill_common::wac::WAC_SUSPEND_READY.store(true, std::sync::atomic::Ordering::Relaxed); + } + // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration, None)); + return Ok((job_id, duration)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1104,7 +1121,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration, wac_job_ids)) + Ok((completed_job.id, duration)) } async fn commit_completed_job( @@ -1122,7 +1139,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, Option)> { +) -> windmill_common::error::Result<(Option, i64, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1252,74 +1269,23 @@ async fn commit_completed_job( .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; } - let mut wac_job_ids: Option = None; + // Before `delete_job`: the parent's rows are locked ahead of the child's own + // queue row (see `record_child_completion` for the order this must keep). + let mut wac_parent_ready = false; if !completed_job.is_flow_step() { if let Some(parent_job) = completed_job.parent_job { - // Only update WAC parents (v1 or v2). The WHERE condition skips - // non-WAC parents entirely (error handlers, run_script children, etc.). - // Also returns pending_steps.job_ids so WAC v2 child completion - // doesn't need a separate read. - let row = sqlx::query_scalar!( - r#"UPDATE v2_job_status SET - workflow_as_code_status = jsonb_set( - jsonb_set( - workflow_as_code_status, - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'duration_ms'], - to_jsonb($2::bigint) - ) - WHERE id = $3 AND workflow_as_code_status IS NOT NULL - RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#, - &completed_job.id.to_string(), + wac_parent_ready = windmill_common::wac::record_child_completion( + &mut tx, + &parent_job, + &completed_job.id, + success, duration, - parent_job + sanitized_result.as_ref(), ) - .fetch_optional(&mut *tx) .warn_after_seconds(10) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update parent job `duration_ms` in workflow as code status: {}", - e, - ) - }) - .ok() - .flatten(); - wac_job_ids = row.flatten(); - - // If parent was already completed (e.g. cancelled), update v2_job_completed instead - if wac_job_ids.is_none() { - let _ = sqlx::query!( - r#"UPDATE v2_job_completed SET - workflow_as_code_status = jsonb_set( - jsonb_set( - workflow_as_code_status, - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'duration_ms'], - to_jsonb($2::bigint) - ) - WHERE id = $3 AND workflow_as_code_status IS NOT NULL"#, - &completed_job.id.to_string(), - duration, - parent_job - ) - .execute(&mut *tx) - .warn_after_seconds(10) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update completed parent job `duration_ms` in workflow as code status: {}", - e, - ) - }); - } + .await?; } } - // tracing::error!("Added completed job {:#?}", queued_job); let mut _skip_downstream_error_handlers = false; tx = delete_job(tx, &job_id).warn_after_seconds(10).await?; @@ -1547,14 +1513,19 @@ async fn commit_completed_job( completed_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids)) + Ok(( + None, + duration, + _skip_downstream_error_handlers, + wac_parent_ready, + )) } async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, Option), Error>> { +) -> Option, i64, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { @@ -2068,10 +2039,35 @@ fn apply_completed_job_cloud_usage( queued_job: &MiniCompletedJob, _duration: i64, ) { - if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { + if !queued_job.is_flow() { + meter_execution_seconds( + db, + &queued_job.workspace_id, + &queued_job.permissioned_as_email, + _duration, + ); + } +} + +/// Charge `_duration` of execution time to the cloud usage meters: the workspace's +/// monthly row, plus the per-user row on non-premium plans. +/// +/// The unit is one finished **segment**, not one job. A Workflow-as-Code parent parks on +/// a sleep, an approval or its children and resumes with a fresh timer, so its compute +/// arrives here as several calls; metering only the one at completion would drop +/// everything it ran before its first park. +/// +/// Fire-and-forget, like every other write to `usage`: billing must never hold up the +/// job that produced it. +/// +/// `w_id` and `email` are billed as given and authorize nothing on their own — take them +/// from a job the caller already holds, never from request input. +#[cfg(feature = "cloud")] +pub fn meter_execution_seconds(db: &Pool, w_id: &str, email: &str, _duration: i64) { + if *CLOUD_HOSTED && _duration > 1000 { let db = db.clone(); - let w_id = queued_job.workspace_id.clone(); - let email = queued_job.permissioned_as_email.clone(); + let w_id = w_id.to_string(); + let email = email.to_string(); let w_id2 = w_id.clone(); let email2 = email.clone(); tokio::task::spawn(async move { @@ -4127,33 +4123,10 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { { let maybe_new_id = match j.kind { - JobKind::Dependencies => { - let deployment_message = j - .args - .clone() - .map(|hashmap| { - hashmap - .get("deployment_message") - .map(|map_value| serde_json::from_str::(map_value.get()).ok()) - .flatten() - }) - .flatten(); - - // This way we tell downstream which script we should archive when the resolution is finished. - // (not used at the moment) - j.args - .as_mut() - .map(|args| args.insert("base_hash".to_owned(), to_raw_value(&*base_hash))); - - windmill_common::scripts::clone_script( - j.runnable_path(), - &j.workspace_id, - deployment_message, - db, - ) - .await? - .new_hash - } + // A script gets its new version from the worker, once the generated lock is known + // to differ from the live version's: minting one here would deploy, and walk the + // importers of, a version whose lock turns out byte-identical to its parent's. + JobKind::Dependencies => *base_hash, JobKind::FlowDependencies => { sqlx::query_scalar!( "INSERT INTO flow_version @@ -4791,6 +4764,36 @@ pub async fn check_debouncing_within_limits( } } +/// Whether the tag's queue name is computed from the job's arguments, so that a caller holding +/// arguments it could not build knows the tag cannot be built either. +pub fn tag_reads_args(tag: &str) -> bool { + RE_ARG_TAG.is_match(tag) +} + +/// Whether the tag reads the flow's state (`$flow_expr[results.a.foo]`), which only the flow +/// runtime can resolve, right before pushing the step. A malformed placeholder counts too, so it +/// is rejected or dropped instead of queueing the job on its literal text. +pub fn tag_reads_flow_expr(tag: &str) -> bool { + tag.contains("$flow_expr[") +} + +/// Renders the value at the dotted `path` below `root` as a dynamic tag component, shared by +/// `$args[...]` and `$flow_expr[...]`: its JSON text with surrounding quotes trimmed, and empty +/// once a segment is missing. Only object keys are followed, never array indexes. +pub fn render_tag_path(root: Option<&RawValue>, path: &str) -> String { + let mut value = root.map(|x| x.get()).unwrap_or_default().to_string(); + for part in path.split('.').filter(|p| !p.is_empty()) { + match serde_json::from_str::(&value) { + Ok(obj) => value = obj.get(part).map(|v| v.to_string()).unwrap_or_default(), + Err(_) => { + value = String::new(); + break; + } + } + } + value.trim_matches('"').to_string() +} + pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String { // Save this value to avoid parsing twice let workspaced = x.as_str().replace("$workspace", workspace_id).to_string(); @@ -4798,40 +4801,12 @@ pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> Strin let mut interpolated = workspaced.clone(); for cap in RE_ARG_TAG.captures_iter(&workspaced) { let arg_name = cap.get(1).unwrap().as_str(); - let arg_value = if arg_name.contains('.') { - let parts: Vec<&str> = arg_name.split('.').collect(); - let root = parts[0]; - let mut value = args - .args - .get(root) - .or(args.extra.as_ref().and_then(|x| x.get(root))) - .map(|x| x.get()) - .unwrap_or_default() - .to_string(); - - for part in parts.iter().skip(1) { - if let Ok(obj) = serde_json::from_str::(&value) { - value = obj - .get(part) - .and_then(|v| Some(v.to_string())) - .unwrap_or_default() - .as_str() - .to_string(); - } else { - value = "".to_string(); // Invalid JSON or missing field - break; - } - } - value.trim_matches('"').to_string() - } else { - args.args - .get(arg_name) - .or(args.extra.as_ref().and_then(|x| x.get(arg_name))) - .map(|x| x.get()) - .unwrap_or_default() - .trim_matches('"') - .to_string() - }; + let (root, rest) = arg_name.split_once('.').unwrap_or((arg_name, "")); + let root_value = args + .args + .get(root) + .or(args.extra.as_ref().and_then(|x| x.get(root))); + let arg_value = render_tag_path(root_value.map(|x| &**x), rest); interpolated = interpolated.replace(format!("$args[{}]", arg_name).as_str(), &arg_value); } @@ -5410,6 +5385,8 @@ pub fn empty_result() -> Box { lazy_static::lazy_static! { pub static ref RE_ARG_TAG: Regex = Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); + pub static ref RE_FLOW_EXPR_TAG: Regex = + Regex::new(r#"\$flow_expr\[((?:\w+\.)*\w+)\]"#).unwrap(); } #[cfg(feature = "cloud")] @@ -5617,6 +5594,8 @@ async fn push_inner<'c, 'd>( ) { // Check current usage with SELECT (fast, no row locks) // Only check user usage for non-premium workspaces + // `email` here and in the per-user checks below can be a cached dispatch address, up + // to one notify poll stale; accepted, see `get_email_from_permissioned_as`. let (current_workspace_usage, current_user_usage) = check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?; @@ -6677,7 +6656,10 @@ async fn push_inner<'c, 'd>( ); windmill_common::worker::dedicated_worker_tag(workspace_id, &full_path) } else { - if tag == Some("".to_string()) { + // The flow runtime resolves a step's `$flow_expr[...]` before pushing it, so one still here + // was pushed with no flow state to read (a step test, a dependency job) and would name a + // queue no worker serves: the job runs on its default tag instead. + if tag == Some("".to_string()) || tag.as_deref().is_some_and(tag_reads_flow_expr) { tag = None; } @@ -7007,7 +6989,11 @@ async fn push_inner<'c, 'd>( language as Option, same_worker, pre_run_error.map(|e| e.to_string()), - email, + // `job_authed`'s, not the handed-in `email`: unless the caller's own authed already names + // this identity, it came through `fetch_authed_from_permissioned_as`, which re-resolves the + // address from the principal's live binding. The same statement writes it to + // `job_perms.email`, and the two columns naming different accounts is what this prevents. + job_authed.email, visible_to_owner, flow_innermost_root_job, guarded_concurrent_limit, @@ -7124,7 +7110,8 @@ async fn push_inner<'c, 'd>( hm.insert("created_by", user); } let audit_author = AuditAuthor { - email: email.to_string(), + // `job_authed`'s address, matching `v2_job` and `job_perms` above. + email: job_authed.email.clone(), username: if runs_on_behalf { windmill_common::auth::permissioned_as_to_username(&permissioned_as) } else { @@ -7421,15 +7408,19 @@ async fn check_workspace_queue_cap<'c>( // Ok(()) // } -pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value { - let reason = job - .canceled_reason - .as_deref() - .unwrap_or_else(|| "no reason given"); - let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown"); +/// The result payload a job cancelled anywhere carries. Callers that hold the cancel +/// outside a `MiniPulledJob` — a row read after the pull, say — go through this rather +/// than rebuilding the shape. +pub fn canceled_result(reason: Option<&str>, canceler: Option<&str>) -> serde_json::Value { + let reason = reason.unwrap_or("no reason given"); + let canceler = canceler.unwrap_or("unknown"); serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler}) } +pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value { + canceled_result(job.canceled_reason.as_deref(), job.canceled_by.as_deref()) +} + /// Helper function to create a restarted module for branch/iteration restart fn create_restarted_module( module: &FlowStatusModule, @@ -8020,3 +8011,76 @@ mod git_sync_concurrency_key_tests { assert!(a.len() <= 255 && b.len() <= 255); } } + +#[cfg(test)] +mod result_metadata_tests { + use super::{ResultMetadata, ValidableJson}; + use serde_json::value::RawValue; + + fn from_raw(json: &str) -> ResultMetadata { + RawValue::from_string(json.to_string()) + .unwrap() + .result_metadata() + } + + fn from_value(json: &str) -> ResultMetadata { + serde_json::from_str::(json) + .unwrap() + .result_metadata() + } + + #[test] + fn array_result_carries_no_markers() { + for json in [r#"[["label"], "boom"]"#, r#"[null, "boom"]"#] { + for meta in [from_raw(json), from_value(json)] { + assert!( + meta.wm_labels.is_none() && meta.wm_failure.is_none(), + "{json}" + ); + } + } + let meta = from_raw(r#"{"wm_labels": ["label"], "wm_failure": "boom"}"#); + assert_eq!(meta.wm_labels, Some(vec!["label".to_string()])); + assert_eq!(meta.wm_failure.as_deref(), Some("boom")); + } +} + +#[cfg(test)] +mod render_tag_path_tests { + use super::{interpolate_args, render_tag_path, PushArgs}; + use serde_json::value::RawValue; + use std::collections::HashMap; + + fn render(root: &str, path: &str) -> String { + render_tag_path( + Some(&RawValue::from_string(root.to_string()).unwrap()), + path, + ) + } + + // Existing `$args[...]` tags route on exactly these renderings. + #[test] + fn renders_like_args_tags() { + assert_eq!(render(r#""eu""#, ""), "eu"); + assert_eq!(render(r#"{"a": {"b": "eu"}}"#, "a.b"), "eu"); + assert_eq!(render(r#"{"n": 4}"#, "n"), "4"); + assert_eq!(render("null", ""), "null"); + assert_eq!(render(r#"{"a": 1}"#, "b.c"), ""); + assert_eq!(render(r#"{"a": ["eu"]}"#, "a.0"), ""); + assert_eq!(render_tag_path(None, "a"), ""); + + let args = HashMap::from([("cfg".to_string(), raw(r#"{"lang": "eu"}"#))]); + let push_args = PushArgs { + args: &args, + extra: Some(HashMap::from([("e".to_string(), raw(r#""x""#))])), + }; + assert_eq!( + interpolate_args("w-$args[cfg.lang]-$args[e]".to_string(), &push_args, "ws"), + "w-eu-x" + ); + } + + fn raw(json: &str) -> Box { + RawValue::from_string(json.to_string()).unwrap() + } +} diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index fa495c9cd2..5d5cf005fc 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -166,13 +166,10 @@ pub async fn push_scheduled_job<'c>( } }; - let next = sched.find_next(&starting_from); - // println!("next event ({:?}): {}", tz, next); - // println!("next event(UTC): {}", next.with_timezone(&chrono::Utc)); + let next = sched.find_next(&starting_from)?; // Scheduled events must be stored in the database in UTC let next = next.with_timezone(&chrono::Utc); - // panic!("next: {}", next); let already_exists: bool = sqlx::query_scalar!( // Query plan: // - use of the `ix_v2_job_root_by_path` index; hence the `parent_job IS NULL` clause. diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index d58d9a5a46..333d211765 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -921,6 +921,47 @@ mod schedule_push { Ok(()) } + // ----------------------------------------------------------------------- + // try_schedule_next_job: a cron with no run left disables the schedule + // ----------------------------------------------------------------------- + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_cron_with_no_run_left_disables_schedule( + db: Pool, + ) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap, permissioned_as, cron_version) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 9 1 1 * 2020', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false, 'u/test-user', 'v1')" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.schedule = "0 0 9 1 1 * 2020".to_string(); + s.cron_version = Some("v1".to_string()); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = + try_schedule_next_job(&db, tx, &job, &schedule, &schedule.script_path).await; + assert!(err.is_none(), "completion must go through, got: {err:?}"); + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 0); + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(!enabled, "schedule with no run left must be disabled"); + assert!( + error.as_deref().is_some_and(|e| e.contains("no run left")), + "error should say why, got: {error:?}" + ); + Ok(()) + } + // ----------------------------------------------------------------------- // try_schedule_next_job: disabled schedule leaves no side effects // ----------------------------------------------------------------------- diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 6e80b1405e..750bf7ebac 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -26,7 +26,7 @@ use std::{ cell::RefCell, path::PathBuf, rc::Rc, - sync::{Arc, Mutex}, + sync::{Arc, LazyLock, Mutex}, }; // Re-export deno_telemetry for use by windmill-worker's otel proxy @@ -182,10 +182,33 @@ struct LogString { pub s: mpsc::UnboundedSender, } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct NativeAnnotation { pub useragent: Option, pub proxy: Option<(String, Option<(String, String)>)>, + /// `//fetch_response_timeout `: per-script override of + /// [`default_fetch_response_timeout_secs`]. `Some(0)` disables it for this + /// script; `None` leaves the default in force. + pub fetch_response_timeout_secs: Option, +} + +/// How long `fetch()` waits for a response to begin, in seconds; `0` disables. +/// +/// Covers everything up to the response headers and stops there, so a body may +/// then stream for any length of time. `src/runtime.js` holds the semantics. +/// +/// Must exceed `TIMEOUT_WAIT_RESULT` (default 600), which holds synchronous job +/// calls open without headers. Raising that hot-reloaded instance setting may +/// also require raising `WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS` in the deployment +/// and restarting workers: this environment value is cached for the process. +pub fn default_fetch_response_timeout_secs() -> u64 { + static SECS: LazyLock = LazyLock::new(|| { + std::env::var("WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS") + .ok() + .and_then(|x| x.trim().parse::().ok()) + .unwrap_or(900) + }); + *SECS } /// Serializes V8 isolate creation as defense-in-depth against concurrent @@ -401,7 +424,7 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { } pub fn get_annotation(inner_content: &str) -> NativeAnnotation { - let mut res = NativeAnnotation { useragent: None, proxy: None }; + let mut res = NativeAnnotation::default(); let anns = inner_content .lines() @@ -414,6 +437,13 @@ pub fn get_annotation(inner_content: &str) -> NativeAnnotation { res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); } else if ann.starts_with("proxy") { res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); + } else if ann.starts_with("fetch_response_timeout") { + // A typo falls back to the default, never to "no timeout". + res.fetch_response_timeout_secs = ann + .trim_start_matches("fetch_response_timeout") + .trim() + .parse::() + .ok(); } } res @@ -554,6 +584,16 @@ pub(crate) fn create_nativets_runtime( let ops = vec![op_get_static_args(), op_log()]; let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; + // deno_web's setTimeout puts its delay through `webidl.converters.long`, + // which wraps at 32 bits: past i32::MAX ms (~24.8 days) the delay comes out + // negative and fires immediately, so an over-generous setting would abort + // every fetch on the spot. Cap rather than wrap. + let fetch_response_timeout_ms = ann + .fetch_response_timeout_secs + .unwrap_or_else(default_fetch_response_timeout_secs) + .saturating_mul(1000) + .min(i32::MAX as u64); + let fetch_options = deno_fetch::Options { root_cert_store_provider: NATIVE_ROOT_CERT_STORE_PROVIDER.clone(), user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), @@ -625,10 +665,15 @@ pub(crate) fn create_nativets_runtime( } // Per-isolate JS init that can't run in the snapshot (runtime.js executes at - // snapshot-build time): currently seeds performance.timeOrigin via - // setTimeOrigin(), which must read this isolate's wall clock. + // snapshot-build time): the wall clock behind performance.timeOrigin and the + // fetch response timeout are both per-isolate values. js_runtime - .execute_script("", "globalThis.__wmInitPerIsolate()") + .execute_script( + "", + format!( + "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})" + ), + ) .map_err(windmill_common::error::to_anyhow)?; Ok(CreatedRuntime { js_runtime, log_receiver, memory_limit_rx }) @@ -793,6 +838,11 @@ pub async fn eval_fetch_timeout( } } let w_id_for_tracing = w_id_for_tracing; + // nativets delivers logs in-process, so they never reach the masking in + // `handle_child::write_lines` and a `console.log` of `$WM_TOKEN` would be + // persisted verbatim. Mask here rather than in the detached task draining into + // `append_logs`: this loop normally runs while the job is still registered. + let mut masker = windmill_common::sensitive_log_masks::JobMasker::new(job_id); let handle = tokio::spawn(async move { let mut result_stream = String::new(); let mut is_stream = false; @@ -800,10 +850,20 @@ pub async fn eval_fetch_timeout( use windmill_common::result_stream::extract_stream_from_logs; use windmill_common::tracing_init::{OTEL_JOB_LOGS, OTEL_PREFIX}; + let stream = extract_stream_from_logs(&log.trim_end_matches("\n")); + + // A stream chunk is result data, not a log line — it never reaches + // `job_logs`, and `merge_result_stream` can make it the job's result — + // so it stays raw wherever it goes, here and in the mirror below. + // Deliberately unlike `handle_child`, which streams the masked text. + // Routed before masking because the notice is one-shot: spent on a chunk + // no sink persists, a later redaction in `job_logs` would go unexplained. + let logged = stream.is_none().then(|| masker.mask(&log).into_owned()); + // Mirror `process_streaming_log_lines` (EE) + the OTEL_JOB_LOGS // hook from handle_child.rs, neither of which runs for nativets // since nativets delivers logs in-process via the log channel. - for line in log.lines() { + for line in logged.as_deref().unwrap_or(&log).lines() { tracing::info!( target: "windmill:job_log", job_id = ?job_id, @@ -817,7 +877,7 @@ pub async fn eval_fetch_timeout( } } - if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { + if let Some(stream) = stream { if !is_stream { is_stream = true; if let Some(ref f) = stream_notifier_update { @@ -829,8 +889,8 @@ pub async fn eval_fetch_timeout( if let Err(e) = result_stream_sender.send(stream) { tracing::error!("failed to send result stream: {e}"); } - } else { - if let Err(e) = append_logs_sender.send(log) { + } else if let Some(logged) = logged { + if let Err(e) = append_logs_sender.send(logged) { tracing::error!("failed to send log: {e}"); } } diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js index 01bec09983..88e10f765a 100644 --- a/backend/windmill-runtime-nativets/src/runtime.js +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -30,9 +30,115 @@ import * as performance from "ext:deno_web/15_performance.js"; import "ext:deno_web/16_image_data.js"; import "ext:deno_fetch/27_eventsource.js"; +// deno_fetch applies no deadline, so a peer that accepts a request and then +// never answers leaves `await fetch(...)` pending until the job timeout, which +// self-hosted defaults to 7 days. +const ORIGINAL_FETCH = fetch.fetch; + +// deno_web's timers reject any `this` other than undefined/globalThis, so +// `timers.setTimeout(...)` passes the module namespace and throws "Illegal +// invocation". +const setTimeoutUnbound = timers.setTimeout; +const clearTimeoutUnbound = timers.clearTimeout; +// Captured before user code shares the isolate and could redefine them, the +// way deno's own modules reach intrinsics through primordials. +const PromiseReject = Promise.reject.bind(Promise); +const promiseThen = Function.prototype.call.bind(Promise.prototype.then); +const abortSignalAny = abortSignal.AbortSignal.any.bind(abortSignal.AbortSignal); +const abortControllerAbort = Function.prototype.call.bind( + abortSignal.AbortController.prototype.abort, +); +const ReflectApply = Reflect.apply; + +// Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop +// for the impossible case of fetch running before that init. +let fetchResponseTimeoutMs = 900_000; + +function fetchResponseTimeoutError(requestUrl, timeoutMs) { + let target; + try { + const parsed = new url.URL(requestUrl); + // Query and fragment routinely carry tokens, and this reaches a job log. + target = parsed.origin + parsed.pathname; + } catch { + target = "the request target"; + } + return new domException.DOMException( + `fetch to ${target} timed out: no response headers arrived within ` + + `${Math.round(timeoutMs / 1000)}s (this covers connect, request upload ` + + `and the wait for the server to start replying; once a response begins ` + + `it is never interrupted). Change it per script with ` + + `"//fetch_response_timeout " (0 disables), or instance-wide ` + + `with WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS.`, + "TimeoutError", + ); +} + globalThis.atob = base64.atob; globalThis.btoa = base64.btoa; -globalThis.fetch = fetch.fetch; +// Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT +// pins that an aborted fetch settles in the same tick, which adopting its +// promise through another one would break. Construction still has to reject +// rather than throw, so it is caught and handed back as a rejection. +globalThis.fetch = function fetch(input, init = undefined) { + const timeoutMs = fetchResponseTimeoutMs; + // Forwarded with the original argument count, so deno still sees an empty + // call as empty and raises its own "1 argument required". The default on + // `init` is what keeps `fetch.length` at 1, as the standard has it. + if (!(timeoutMs > 0) || arguments.length < 1) { + return ReflectApply(ORIGINAL_FETCH, undefined, arguments); + } + + let req; + let controller; + let signal; + try { + // RequestInit is a WebIDL dictionary: copying it drops inherited and + // non-enumerable members, and inheriting from it runs accessors against the + // wrong receiver. Hand it to the same Request constructor fetch() would, + // and carry our own signal in an init of our own. + req = new request.Request(input, init); + + // `req.signal` is deno's own resolution of init.signal over an input + // Request's signal, so combining with it preserves the caller's abort and + // reason while ours only adds a ceiling. + controller = new abortSignal.AbortController(); + signal = abortSignalAny([req.signal, controller.signal]); + } catch (e) { + return PromiseReject(e); + } + + // Already aborted: hand back deno's own settled rejection untouched, and arm + // nothing -- there is no response to wait for. + if (signal.aborted) { + return ORIGINAL_FETCH(req, { signal }); + } + + let timer = setTimeoutUnbound(() => { + timer = undefined; + abortControllerAbort(controller, fetchResponseTimeoutError(req.url, timeoutMs)); + }, timeoutMs); + // Disarmed on headers, never on body completion: a response that has begun + // arriving must be free to stream for as long as it needs. + const disarm = () => { + if (timer !== undefined) { + clearTimeoutUnbound(timer); + timer = undefined; + } + }; + + return promiseThen( + ORIGINAL_FETCH(req, { signal }), + (res) => { + disarm(); + return res; + }, + (e) => { + disarm(); + throw e; + }, + ); +}; globalThis.Request = request.Request; globalThis.Response = response.Response; globalThis.Blob = file.Blob; @@ -123,7 +229,11 @@ Object.assign(globalThis, { // Per-isolate init, invoked from Rust after the snapshot is restored (this // module body runs at snapshot-build time, not per isolate). -globalThis.__wmInitPerIsolate = () => { +globalThis.__wmInitPerIsolate = (config) => { + if (config != null && typeof config.fetchResponseTimeoutMs === "number") { + fetchResponseTimeoutMs = config.fetchResponseTimeoutMs; + } + // setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock; // without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN. performance.setTimeOrigin(); diff --git a/backend/windmill-runtime-nativets/src/smoke_tests.rs b/backend/windmill-runtime-nativets/src/smoke_tests.rs index b581a9f1fe..f3da460227 100644 --- a/backend/windmill-runtime-nativets/src/smoke_tests.rs +++ b/backend/windmill-runtime-nativets/src/smoke_tests.rs @@ -24,7 +24,7 @@ use crate::{transpile_ts, NativeAnnotation, PrewarmedIsolate, PrewarmedResult}; /// positional args, and return the isolate's result + captured logs. async fn run_ts(ts: &str, arg_names: &[&str], args: serde_json::Value) -> PrewarmedResult { let js = transpile_ts(ts.to_string()).expect("transpile_ts failed"); - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let arg_names: Vec = arg_names.iter().map(|s| s.to_string()).collect(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, arg_names, None); iso.wait_ready().await.expect("isolate failed to pre-warm"); @@ -232,7 +232,7 @@ export async function main(i: number): Promise { for i in 0..N { let js = js.clone(); let h = tokio::spawn(async move { - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec!["i".to_string()], None); iso.wait_ready().await.expect("pre-warm failed"); diff --git a/backend/windmill-runtime-nativets/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js index 7c7d649042..c2dbd1fef1 100644 --- a/backend/windmill-runtime-nativets/src/windmill-client.js +++ b/backend/windmill-runtime-nativets/src/windmill-client.js @@ -6457,7 +6457,7 @@ var AppService = class { * @param data.workspace * @param data.path * @param data.requestBody update app - * @returns string app updated + * @returns AppDeployed the path the app now lives at and the version this deploy wrote * @throws ApiError */ static updateApp(data) { diff --git a/backend/windmill-runtime-nativets/tests/annotation.rs b/backend/windmill-runtime-nativets/tests/annotation.rs new file mode 100644 index 0000000000..d8319a87a5 --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/annotation.rs @@ -0,0 +1,37 @@ +//! `//fetch_response_timeout` parsing. Cheap (no V8), so it stays out of the +//! e2e file. + +use windmill_runtime_nativets::get_annotation; + +#[test] +fn a_value_is_parsed_and_zero_stays_distinct_from_absent() { + // Collapsing `Some(0)` to `None` would silently reinstate the default on a + // script that explicitly asked for no limit. + assert_eq!( + get_annotation("//native\n//fetch_response_timeout 30\n").fetch_response_timeout_secs, + Some(30) + ); + assert_eq!( + get_annotation("//fetch_response_timeout 0\n").fetch_response_timeout_secs, + Some(0) + ); + assert_eq!( + get_annotation("//native\n").fetch_response_timeout_secs, + None + ); +} + +#[test] +fn a_malformed_value_falls_back_to_the_default_not_to_no_timeout() { + for src in [ + "//fetch_response_timeout abc\n", + "//fetch_response_timeout\n", + "//fetch_response_timeout -5\n", + ] { + assert_eq!( + get_annotation(src).fetch_response_timeout_secs, + None, + "{src:?} should leave the default in force" + ); + } +} diff --git a/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs b/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs new file mode 100644 index 0000000000..5118b5f93f --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/fetch_response_timeout.rs @@ -0,0 +1,593 @@ +//! The nativets `fetch()` response timeout. +//! +//! Two halves of one contract, and a fix that satisfies only the first is worse +//! than no fix: +//! +//! 1. a peer that accepts a request and never answers is given up on +//! 2. a response that has begun arriving is never cut off, however long it +//! takes in total +//! +//! (2) rules out the obvious implementation — `AbortSignal.timeout(N)` around +//! every fetch would satisfy (1) and break every streaming response and long +//! download. +//! +//! Hermetic: loopback listeners, no egress. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::Mutex; + +use windmill_runtime_nativets::{transpile_ts, NativeAnnotation, PrewarmedIsolate}; + +/// The annotation is in whole seconds, so the tests scale around this. +const TIMEOUT_SECS: u64 = 2; + +async fn run_with_timeout_secs(ts: &str, secs: u64) -> Result { + let js = transpile_ts(ts.to_string()).expect("transpile_ts failed"); + let ann = + NativeAnnotation { fetch_response_timeout_secs: Some(secs), ..NativeAnnotation::default() }; + let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None); + iso.wait_ready().await.expect("isolate failed to pre-warm"); + let res = iso + .start_execution("{}".to_string()) + .wait() + .await + .expect("isolate panicked"); + res.result.map(|raw| raw.get().to_string()) +} + +/// A peer that reads the request and then answers nothing, closing only after +/// `close_after` so no test leaves an isolate wedged on a pending fetch. +/// +/// The socket is held rather than dropped on accept: dropping it sends a FIN, +/// which surfaces as a connection error — the easy failure, not this one. +async fn spawn_silent_peer(seen: Arc>>, close_after: Duration) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + if let Ok(n) = sock.read(&mut buf).await { + seen.lock().await.extend_from_slice(&buf[..n]); + } + tokio::time::sleep(close_after).await; + drop(sock); + }); + } + }); + port +} + +/// A peer that records the request it received and answers 200 immediately. +async fn spawn_echo_peer(seen: Arc>>) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + if let Ok(n) = sock.read(&mut buf).await { + seen.lock().await.extend_from_slice(&buf[..n]); + } + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + }); + } + }); + port +} + +/// A peer that responds after `headers_after`, then dribbles a chunked body out +/// over `chunks * chunk_every`. +async fn spawn_streaming_peer( + headers_after: Duration, + chunks: usize, + chunk_every: Duration, +) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buf = [0u8; 8192]; + let _ = sock.read(&mut buf).await; + tokio::time::sleep(headers_after).await; + if sock + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .await + .is_err() + { + return; + } + for _ in 0..chunks { + tokio::time::sleep(chunk_every).await; + if sock.write_all(b"1\r\nx\r\n").await.is_err() { + return; + } + } + let _ = sock.write_all(b"0\r\n\r\n").await; + }); + } + }); + port +} + +const POST_TO_SILENT_PEER: &str = r#" +export async function main(): Promise { + const res = await fetch("http://127.0.0.1:{port}/orders", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: 1 }), + }); + return res.status; +} +"#; + +fn post_script(port: u16) -> String { + POST_TO_SILENT_PEER.replace("{port}", &port.to_string()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_peer_that_never_answers_is_given_up_on() { + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen.clone(), Duration::from_secs(60)).await; + + let started = Instant::now(); + let err = run_with_timeout_secs(&post_script(port), TIMEOUT_SECS) + .await + .expect_err("fetch against a peer that never answers must not resolve"); + let elapsed = started.elapsed(); + + assert!( + elapsed >= Duration::from_secs(TIMEOUT_SECS), + "gave up after {elapsed:?}, before the configured {TIMEOUT_SECS}s -- \ + the timeout is firing on something other than the wait for a response", + ); + assert!( + elapsed < Duration::from_secs(TIMEOUT_SECS + 15), + "took {elapsed:?} to give up", + ); + + // The message has to stand on its own in a job log: a hung request is + // otherwise indistinguishable from a slow one. + assert!( + err.contains("no response headers arrived within"), + "error should explain what timed out, got: {err}", + ); + assert!( + err.contains("/orders") && err.contains("fetch_response_timeout"), + "error should name the target and how to change the limit, got: {err}", + ); + + // Without this the test would also pass if the request never went out. + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.contains("POST /orders"), + "peer should have received the request, got: {body}", + ); +} + +/// The counterfactual for the test above: with the timeout disabled, the same +/// script against the same peer is still running well past the point the +/// timeout would have fired. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn without_a_timeout_the_same_request_keeps_running() { + let seen = Arc::new(Mutex::new(Vec::new())); + // Closes eventually, so the isolate unwinds instead of pinning a blocking + // task for the rest of the test binary's life. + let port = spawn_silent_peer(seen, Duration::from_secs(TIMEOUT_SECS * 3)).await; + + let still_running = tokio::time::timeout( + Duration::from_secs(TIMEOUT_SECS * 2), + run_with_timeout_secs(&post_script(port), 0), + ) + .await + .is_err(); + + assert!( + still_running, + "with the timeout disabled the request should still have been pending \ + at {}s -- if it ends on its own, the test above proves nothing", + TIMEOUT_SECS * 2, + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_streaming_body_outliving_the_timeout_is_not_cut_off() { + // Headers land fast, then the body trickles well past the timeout. A + // total-duration timeout fails here; that is the point of the test. + let port = + spawn_streaming_peer(Duration::from_millis(200), 10, Duration::from_millis(500)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/stream"); + return `${{res.status}}:${{(await res.text()).length}}`; +}} +"# + ); + + let started = Instant::now(); + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a streaming response must not be interrupted"); + let elapsed = started.elapsed(); + + assert_eq!(out, "\"200:10\"", "full body should arrive intact"); + assert!( + elapsed > Duration::from_secs(TIMEOUT_SECS), + "the transfer ({elapsed:?}) has to outlast the {TIMEOUT_SECS}s timeout \ + for this to be exercising anything", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_slow_but_answering_peer_is_not_cut_off() { + // A quarter of the window rather than half: CI runs this at + // --test-threads=10 alongside other V8 isolates, and this margin is what + // absorbs executor starvation. + let port = spawn_streaming_peer( + Duration::from_millis(1_000 * TIMEOUT_SECS / 4), + 1, + Duration::from_millis(10), + ) + .await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/slow"); + await res.text(); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a slow but answering peer must not be cut off"); + assert_eq!(out, "200"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_over_large_timeout_does_not_wrap_into_an_instant_one() { + // deno_web's setTimeout puts its delay through `webidl.converters.long`, + // which wraps at 32 bits. Unclamped, this ~46-day setting wraps negative and + // aborts immediately -- asking for a longer leash would kill every fetch. + let port = spawn_streaming_peer(Duration::from_millis(50), 1, Duration::from_millis(10)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/ok"); + await res.text(); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, 4_000_000) + .await + .expect("an over-large timeout must not abort the request"); + assert_eq!(out, "200"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_caller_abort_still_wins_with_its_own_reason() { + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(60)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const ac = new AbortController(); + setTimeout(() => ac.abort(new Error("caller_abort_marker")), 200); + try {{ + await fetch("http://127.0.0.1:{port}/probe", {{ signal: ac.signal }}); + return "unexpectedly resolved"; + }} catch (e) {{ + return String((e as Error).message); + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch its own abort"); + assert!( + out.contains("caller_abort_marker"), + "caller's abort reason should survive being combined with ours, got: {out}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_timeout_is_catchable_as_a_timeout_error() { + // Scripts that retry on transient failures need to recognise this one; + // `TimeoutError` matches AbortSignal.timeout()'s reason. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(60)).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + try {{ + await fetch("http://127.0.0.1:{port}/probe"); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch the timeout"); + assert_eq!(out, "\"TimeoutError\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_init_whose_members_are_inherited_is_not_flattened() { + // RequestInit is a WebIDL dictionary and deno_fetch reads its members with + // plain property gets, so they may sit on the prototype chain or be + // non-enumerable. Object spread copies neither, which would silently + // downgrade this POST to a GET and drop the header. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +declare const Object: any; +export async function main(): Promise {{ + const base = {{ method: "POST", headers: {{ "x-probe": "yes" }} }}; + const res = await fetch("http://127.0.0.1:{port}/inherited", Object.create(base)); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("request should succeed"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("POST /inherited"), + "inherited `method` should survive, got: {body}", + ); + assert!( + body.to_lowercase().contains("x-probe: yes"), + "inherited `headers` should survive, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_non_dictionary_init_still_fails_loudly() { + // deno's dictionary converter throws on a non-object init. Copying members + // into a fresh object instead of inheriting would turn `"POST"` into + // {0:"P",1:"O",...} -- a valid dictionary with ignored keys, i.e. a silent + // GET where the caller used to get a TypeError. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + try {{ + await fetch("http://127.0.0.1:{port}/x", "POST" as any); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should catch the error"); + assert_eq!(out, "\"TypeError\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_accessor_backed_init_reads_against_its_own_receiver() { + // A getter on the init must run with the object it was defined on as `this`, + // or a private field is unreachable and it throws. Carrying the init across + // by inheritance rather than by handing it to Request would break this. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +class Init {{ + #method = "POST"; + #body = "from-private-field"; + get method(): string {{ return this.#method; }} + get body(): string {{ return this.#body; }} +}} +export async function main(): Promise {{ + const res = await fetch("http://127.0.0.1:{port}/accessor", new Init() as any); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("an accessor-backed init must not throw"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("POST /accessor") && body.contains("from-private-field"), + "getter-provided method and body should both reach the wire, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_request_input_keeps_its_body_and_headers() { + // The wrapper builds a Request and hands that to fetch, so the body survives + // one more construction than it used to -- deno proxies it rather than + // consuming it, and this pins that. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen.clone()).await; + + let ts = format!( + r#" +export async function main(): Promise {{ + const req = new Request("http://127.0.0.1:{port}/from-request", {{ + method: "PUT", + headers: {{ "x-probe": "yes" }}, + body: "payload-body", + }}); + const res = await fetch(req); + return res.status; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("a Request input must work"); + assert_eq!(out, "200"); + + let body = String::from_utf8_lossy(&seen.lock().await.clone()).to_string(); + assert!( + body.starts_with("PUT /from-request") + && body.to_lowercase().contains("x-probe: yes") + && body.contains("payload-body"), + "method, headers and body should all survive, got: {body}", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_already_aborted_fetch_settles_in_the_same_tick() { + // deno_fetch keeps its outer fetch non-async and returns an already-settled + // rejection untouched, because WPT pins that an aborted fetch settles in the + // same tick. Adopting it through another promise pushes the rejection behind + // any microtask queued after the call. + let ts = r#" +export async function main(): Promise { + const order: string[] = []; + const ac = new AbortController(); + ac.abort(); + const f = fetch("http://127.0.0.1:1/x", { signal: ac.signal }) + .catch(() => { order.push("fetch"); }); + Promise.resolve().then(() => { order.push("queued-after"); }); + await f; + await new Promise((r) => setTimeout(r, 0)); + return order.join(","); +} +"#; + + let out = run_with_timeout_secs(ts, TIMEOUT_SECS) + .await + .expect("script should run"); + assert_eq!( + out, "\"fetch,queued-after\"", + "the rejection must land before a microtask queued after the call", + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_wrapper_keeps_fetch_s_own_shape() { + // The wrapper is indistinguishable from deno's fetch on three counts a + // script can observe: its arity, the error for an empty call, and not + // depending on a mutable `Promise.prototype.then` the way an ordinary + // property lookup would. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_echo_peer(seen).await; + + let ts = format!( + r#" +declare const Promise: any; +declare const AbortSignal: any; +declare const AbortController: any; +export async function main(): Promise {{ + const arity = (fetch as any).length; + + // The message, not just the type: forwarding two explicit `undefined`s + // would still throw a TypeError, just deno's invalid-URL one instead of + // its required-argument one. + let emptyCall = "resolved"; + try {{ + await (fetch as any)(); + }} catch (e) {{ + emptyCall = (e as Error).message.includes("1 argument required") + ? "required-argument" + : `other(${{(e as Error).message}})`; + }} + + // Patched only across the call: the wrapper reaches for these while + // building its return value, and awaiting under a patched Promise + // prototype would instead measure V8 treating it as a plain thenable. + const originalThen = Promise.prototype.then; + const originalAny = AbortSignal.any; + const originalAbort = AbortController.prototype.abort; + let pending: any; + try {{ + Promise.prototype.then = undefined; + (AbortSignal as any).any = undefined; + (AbortController.prototype as any).abort = undefined; + pending = fetch("http://127.0.0.1:{port}/shape"); + }} finally {{ + Promise.prototype.then = originalThen; + (AbortSignal as any).any = originalAny; + (AbortController.prototype as any).abort = originalAbort; + }} + const status = (await pending).status; + + return `${{arity}}:${{emptyCall}}:${{status}}`; +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("script should run"); + assert_eq!(out, "\"1:required-argument:200\""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_timer_aborts_through_a_captured_intrinsic() { + // The timeout has to fire for this one: `AbortController.prototype.abort` + // is patched out for the whole wait, so an ordinary lookup would throw + // inside the timer callback and leave the request pending forever. + let seen = Arc::new(Mutex::new(Vec::new())); + let port = spawn_silent_peer(seen, Duration::from_secs(TIMEOUT_SECS * 5)).await; + + let ts = format!( + r#" +declare const AbortController: any; +export async function main(): Promise {{ + const originalAbort = AbortController.prototype.abort; + AbortController.prototype.abort = undefined; + try {{ + await fetch("http://127.0.0.1:{port}/patched-abort"); + return "unexpectedly resolved"; + }} catch (e) {{ + return (e as Error).name; + }} finally {{ + AbortController.prototype.abort = originalAbort; + }} +}} +"# + ); + + let out = run_with_timeout_secs(&ts, TIMEOUT_SECS) + .await + .expect("the timeout must still fire"); + assert_eq!(out, "\"TimeoutError\""); +} diff --git a/backend/windmill-runtime-nativets/tests/otel_e2e.rs b/backend/windmill-runtime-nativets/tests/otel_e2e.rs index cadddb2fcf..22fd18de90 100644 --- a/backend/windmill-runtime-nativets/tests/otel_e2e.rs +++ b/backend/windmill-runtime-nativets/tests/otel_e2e.rs @@ -136,7 +136,7 @@ export async function main(): Promise {{ "# ); let js = transpile_ts(ts).expect("transpile failed"); - let ann = NativeAnnotation { useragent: None, proxy: None }; + let ann = NativeAnnotation::default(); let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None); iso.wait_ready().await.expect("isolate failed to pre-warm"); diff --git a/backend/windmill-runtime-nativets/tests/response_timeout_env.rs b/backend/windmill-runtime-nativets/tests/response_timeout_env.rs new file mode 100644 index 0000000000..a206c63134 --- /dev/null +++ b/backend/windmill-runtime-nativets/tests/response_timeout_env.rs @@ -0,0 +1,13 @@ +//! Its own test binary: the setting is read once per process through a +//! `LazyLock`, so nothing else may resolve it first. Adding a second test to +//! this file breaks that isolation. + +use windmill_runtime_nativets::default_fetch_response_timeout_secs; + +#[test] +fn the_env_var_is_what_operators_actually_set() { + // A typo in the variable's name would compile, pass every other test, and + // silently hand every operator the built-in default instead. + std::env::set_var("WINDMILL_FETCH_RESPONSE_TIMEOUT_SECS", "17"); + assert_eq!(default_fetch_response_timeout_secs(), 17); +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index b107d34be4..5bb15b456e 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -7,7 +7,7 @@ */ use dashmap::DashMap; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::IpAddr; use std::sync::LazyLock; @@ -17,6 +17,7 @@ use windmill_api_auth::{ }; use windmill_common::db::DB; use windmill_common::per_minute_counter::PerMinuteCounter; +use windmill_common::ssrf::{private_git_host_allowed, private_git_host_hint, GitRemoteCaller}; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -50,9 +51,9 @@ use windmill_common::{ error::{self, Error, JsonResult, Result}, get_database_url, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only, - fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, WithDraftOverlay, - WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only, fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, + WithDraftOverlay, WithDraftQuery, }, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, @@ -89,6 +90,9 @@ pub fn workspaced_service() -> Router { .route("/git_commit_hash/{*path}", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) + .route("/type/resource_counts", get(list_resource_counts_by_type)) + .route("/type/hub/info", get(list_hub_resource_type_info)) + .route("/type/hub/pick/{name}", post(pick_hub_resource_type)) .route("/type/get/{name}", get(get_resource_type)) .route("/type/exists/{name}", get(exists_resource_type)) .route("/type/update/{name}", post(update_resource_type)) @@ -114,6 +118,10 @@ pub struct ResourceType { pub edited_at: Option>, pub format_extension: Option, pub is_fileset: bool, + /// The name the product goes by (`gsheets` is "Google Sheets"), null where nobody named it. + /// Skipped when absent, so the type files of a synced repo gain nothing until one is set. + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, } #[derive(Deserialize)] @@ -123,6 +131,7 @@ pub struct CreateResourceType { pub description: Option, pub format_extension: Option, pub is_fileset: Option, + pub display_name: Option, } #[derive(Deserialize)] @@ -130,6 +139,22 @@ pub struct EditResourceType { pub schema: Option, pub description: Option, pub is_fileset: Option, + /// Doubly optional so an edit can distinguish the two things a plain + /// `Option` conflates: an absent field leaves the extension alone, while an + /// explicit `null` clears it. A hub pull relies on both — a type that stops + /// being a file type has to stop being one locally too. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub format_extension: Option>, + /// Doubly optional for the same reason. A push from a CLI that predates the field omits it, + /// and must not clear a name the hub set. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -1284,9 +1309,26 @@ async fn create_resource( webhook.send_message( w_id.clone(), - WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, + WebhookMessage::CreateResource { workspace: w_id.clone(), path: resource.path.clone() }, ); + // Trigger CI tests for items that reference this resource + { + let db2 = db.clone(); + let path2 = resource.path.clone(); + let email2 = authed.email.clone(); + let username2 = authed.username.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_dep_map::ci_tests::trigger_ci_tests_for_item( + &db2, &w_id, &path2, "resource", &email2, &username2, + ) + .await + { + tracing::error!(%e, "error triggering CI tests after resource creation"); + } + }); + } + Ok(( StatusCode::CREATED, format!("resource {} created", resource.path), @@ -1303,6 +1345,17 @@ async fn delete_resource( let path = path.to_path(); check_scopes(&authed, || format!("resources:write:{}", path))?; + + // Ahead of the deploy rules: nothing is deployed at a draft-only path, so + // gating this discard on them would strand the row in a protected workspace. + // Ahead of the transaction too — the not-found branch other kinds hang this + // off is the `not_found_if_none` below, past the linked-variable cascade. + if delete_draft_only_for_path(&db, &w_id, UserDraftItemKind::Resource, path, &authed.email) + .await? + { + return Ok(format!("draft-only resource {} deleted", path)); + } + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1314,53 +1367,22 @@ async fn delete_resource( { return Err(Error::PermissionDenied(msg)); } + + let cascade = plan_linked_var_cascade(&db, &w_id, &[path.to_string()]).await?; + let mut tx = user_db.begin(&authed).await?; - // Capture resource data for trashbin before deleting - let trash_resource: Option = sqlx::query_scalar( - "SELECT to_jsonb(t) FROM resource t WHERE path = $1 AND workspace_id = $2", + // The whole row comes back out of the delete, so the trashbin entry below is built from + // what RLS actually removed, and the cascade runs only once RLS has allowed the delete. + let deleted: Option<(String, serde_json::Value)> = sqlx::query_as( + "DELETE FROM resource AS t WHERE t.path = $1 AND t.workspace_id = $2 + RETURNING t.path, to_jsonb(t)", ) .bind(path) .bind(&w_id) .fetch_optional(&mut *tx) .await?; - - // Fetch the resource value before deleting, so we can find linked $var: references - let resource_value: Option> = - sqlx::query_scalar("SELECT value FROM resource WHERE path = $1 AND workspace_id = $2") - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - // Collect all $var: paths referenced in the resource value - let mut linked_var_paths: Vec = Vec::new(); - if let Some(Some(ref value)) = resource_value { - collect_var_refs(value, &mut linked_var_paths); - } - - // A scoped token must not delete linked variables it lacks variables:write for. - check_linked_var_delete_scopes(&authed, &linked_var_paths)?; - - // Capture linked variables for trashbin before deleting them - let trash_linked_vars: Vec = if linked_var_paths.is_empty() { - Vec::new() - } else { - let placeholders: Vec = linked_var_paths - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "SELECT to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path IN ({})", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, serde_json::Value>(&query).bind(&w_id); - for var_path in &linked_var_paths { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; + let (deleted_path, res_data) = not_found_if_none(deleted, "Resource", &path)?; sqlx::query!( "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2", @@ -1370,64 +1392,63 @@ async fn delete_resource( .execute(&mut *tx) .await?; - let deleted_path = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id + let linked_var_paths = cascade.resolve(std::slice::from_ref(&deleted_path)); + + // A scoped token must not delete linked variables it lacks variables:write for. Erroring + // here rolls the resource delete back with it, so nothing is deleted either way. + check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + + // Capture linked variables for trashbin before deleting them + let trash_linked_vars: Vec<(String, serde_json::Value)> = sqlx::query_as( + "SELECT path, to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path = ANY($2)", ) - .fetch_optional(&mut *tx) + .bind(&w_id) + .bind(&linked_var_paths) + .fetch_all(&mut *tx) .await?; - not_found_if_none(deleted_path, "Resource", &path)?; - // Delete linked variables that are actually referenced in the resource value - let deleted_linked_variables: Vec = if linked_var_paths.is_empty() { - Vec::new() - } else { - // Clean up any ws_specific rows for these variables first - // (mark_linked_variables_ws_specific may have auto-inserted them) so - // they don't survive the variable deletion as orphans — a variable - // later recreated at the same path would otherwise inherit the stale - // ws_specific flag. - sqlx::query!( - "DELETE FROM ws_specific - WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", - w_id, - &linked_var_paths - ) - .execute(&mut *tx) - .await?; + let deleted_linked_variables = sqlx::query_scalar!( + "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &linked_var_paths + ) + .fetch_all(&mut *tx) + .await?; - let placeholders: Vec = linked_var_paths - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "DELETE FROM variable WHERE workspace_id = $1 AND path IN ({}) RETURNING path", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, String>(&query).bind(&w_id); - for var_path in &linked_var_paths { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; + // ws_specific has no FK to variable, so a row mark_linked_variables_ws_specific inserted + // would survive as an orphan and a variable later recreated at that path would inherit + // the stale flag. + sqlx::query!( + "DELETE FROM ws_specific + WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + w_id, + &deleted_linked_variables + ) + .execute(&mut *tx) + .await?; - if let Some(res_data) = trash_resource { - let mut trash_data = serde_json::json!({"row": res_data}); - if !trash_linked_vars.is_empty() { - trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); - } - windmill_common::trashbin::move_to_trash( - &mut *tx, - &w_id, - "resource", - path, - trash_data, - &authed.username, - ) - .await?; + // Only the rows that actually went: the snapshot above is what the caller could read, + // which is not necessarily what RLS let it delete, and the trashbin must not hold a copy + // of a secret that is still live. + let trash_linked_vars: Vec = trash_linked_vars + .into_iter() + .filter(|(var_path, _)| deleted_linked_variables.contains(var_path)) + .map(|(_, row)| row) + .collect(); + + let mut trash_data = serde_json::json!({"row": res_data}); + if !trash_linked_vars.is_empty() { + trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); } + windmill_common::trashbin::move_to_trash( + &mut *tx, + &w_id, + "resource", + path, + trash_data, + &authed.username, + ) + .await?; audit_log( &mut *tx, @@ -1439,6 +1460,24 @@ async fn delete_resource( None, ) .await?; + + // The cascade is the one way a variable dies without a variables/delete request of its + // own, so give each one the audit row it would have had, stamped with what took it. + for var_path in &deleted_linked_variables { + let mut params = HashMap::new(); + params.insert("via_resource", path); + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(var_path), + Some(params), + ) + .await?; + } + tx.commit().await?; // Resource gone for everyone: wipe ALL users' drafts at this path (and any linked @@ -1490,35 +1529,205 @@ async fn delete_resource( ); } - Ok(format!("resource {} deleted", path)) + // Name what else went: the cascade is silent from the caller's side otherwise, and a + // secret it took is not something to discover later from a failing job. + if deleted_linked_variables.is_empty() { + Ok(format!("resource {} deleted", path)) + } else { + Ok(format!( + "resource {} deleted, along with its linked variables: {}", + path, + deleted_linked_variables.join(", ") + )) + } } -/// Recursively collect all `$var:path` references from a JSON value. -fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { +/// The forms that resolve a variable path against `variable`, so a value carrying any of them +/// breaks when that variable goes. Only `$var:` is minted by the resource editor, which is why +/// `collect_var_refs` stays narrower than this. +const REFERRER_PREFIXES: [&str; 2] = ["$var:", "$jsonvar:"]; + +/// Recursively collect the variable paths a JSON value references through any of `prefixes`. +fn collect_refs_with_prefixes(value: &serde_json::Value, prefixes: &[&str], out: &mut Vec) { match value { serde_json::Value::String(s) => { - if let Some(var_path) = s.strip_prefix("$var:") { + if let Some(var_path) = prefixes.iter().find_map(|p| s.strip_prefix(p)) { out.push(var_path.to_string()); } } serde_json::Value::Object(m) => { for v in m.values() { - collect_var_refs(v, out); + collect_refs_with_prefixes(v, prefixes, out); } } serde_json::Value::Array(arr) => { for v in arr { - collect_var_refs(v, out); + collect_refs_with_prefixes(v, prefixes, out); } } _ => {} } } -/// Deleting a resource cascades into the `$var:` variables its value references. A -/// scoped token must not use that cascade to delete variables it could not delete -/// directly via `delete_variable` (which gates on `variables:write:`), so require -/// `variables:write` for EVERY linked variable and fail the whole delete otherwise. +/// Recursively collect all `$var:path` references from a JSON value. +fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { + collect_refs_with_prefixes(value, &["$var:"], out) +} + +/// Whether the variable at `var_path` is the resource's own secret rather than one its value +/// merely points at: the same-path twin `delete_variable` and the `update_resource` rename +/// already act on, or `_`, which the connect form mints for a resource +/// type with several secret fields. Anything else is a standalone workspace variable, and +/// deleting one destroys a secret its other referrers still need. +/// +/// A rename moves only the twin, so `_` secrets stop matching and are left +/// behind instead. An orphaned secret can be deleted by hand; a destroyed one cannot. +fn is_owned_linked_var(resource_path: &str, var_path: &str) -> bool { + var_path == resource_path + || var_path + .strip_prefix(resource_path) + .is_some_and(|suffix| suffix.starts_with('_')) +} + +/// Which of `var_paths` a resource outside `excluded_resource_paths` still references. +/// +/// Must run off the non-RLS pool: a referrer in a folder the caller cannot read is precisely +/// the one whose variable has to survive. Nothing from those rows reaches the response. +async fn linked_vars_referenced_elsewhere( + db: &DB, + w_id: &str, + var_paths: &[String], + excluded_resource_paths: &[String], +) -> Result> { + if var_paths.is_empty() { + return Ok(HashSet::new()); + } + // The quotes around the pattern are what make it a whole-JSON-string match rather than a + // prefix one, so `f/db` does not match `"$var:f/db_replica"`. Paths are `proper_id` + // segments, so no JSON escaping or LIKE wildcard can reach this. + let referenced = sqlx::query_scalar!( + "WITH survivors AS ( + SELECT value::text AS rendered FROM resource + WHERE workspace_id = $1 AND NOT (path = ANY($2::text[])) + ) + SELECT v.path FROM unnest($3::text[]) AS v(path) + WHERE EXISTS ( + SELECT 1 FROM survivors s + WHERE strpos(s.rendered, '\"$var:' || v.path || '\"') > 0 + OR strpos(s.rendered, '\"$jsonvar:' || v.path || '\"') > 0 + )", + w_id, + excluded_resource_paths, + var_paths, + ) + .fetch_all(db) + .await?; + Ok(referenced.into_iter().flatten().collect()) +} + +/// A resource delete's candidate cascade, gathered before the caller's transaction opens: the +/// referrer scan runs on `db` because it has to see resources RLS hides, and a second acquire +/// from that same pool under an open `user_db` transaction stalls to the acquire timeout when +/// `DATABASE_CONNECTIONS` is small. `resolve` then decides without touching the database. +/// +/// So a resource that starts referencing a candidate between the scan and the delete keeps a +/// `$var:` pointing at nothing. Narrowing that window means running the scan on the +/// transaction's own connection under a tightly scoped `SET LOCAL ROLE NONE` (the elevation +/// `windmill-queue/src/schedule.rs` uses); closing it needs a lock on every resource write. +struct LinkedVarCascade { + /// Each owned `$var:` path with the requested resource whose value carries it. + candidates: Vec<(String, String)>, + /// Requested resource paths, each with every variable path its value references. + requested: Vec<(String, Vec)>, + /// Candidate paths a resource outside the requested set still references. + referenced_outside: HashSet, +} + +impl LinkedVarCascade { + /// The variables to delete, now that RLS has settled which resources went. + /// + /// A requested resource left standing is the one referrer the scan could not account for, + /// having had to exclude every requested path before RLS had ruled. + fn resolve(&self, deleted_paths: &[String]) -> Vec { + let referenced_by_survivors: HashSet<&str> = self + .requested + .iter() + .filter(|(path, _)| !deleted_paths.contains(path)) + .flat_map(|(_, refs)| refs.iter().map(String::as_str)) + .collect(); + + let mut resolved: Vec = self + .candidates + .iter() + .filter(|(var_path, owner)| { + deleted_paths.contains(owner) + && !self.referenced_outside.contains(var_path.as_str()) + && !referenced_by_survivors.contains(var_path.as_str()) + }) + .map(|(var_path, _)| var_path.clone()) + .collect(); + resolved.sort(); + resolved.dedup(); + resolved + } + + /// Which deleted resource the cascade took `var_path` for, to stamp on its audit row. + fn owner_of<'a>(&'a self, var_path: &str, deleted_paths: &[String]) -> Option<&'a str> { + self.candidates + .iter() + .find(|(candidate, owner)| candidate == var_path && deleted_paths.contains(owner)) + .map(|(_, owner)| owner.as_str()) + } +} + +/// Gather what `LinkedVarCascade::resolve` needs for a delete of `paths`. +async fn plan_linked_var_cascade( + db: &DB, + w_id: &str, + paths: &[String], +) -> Result { + let rows: Vec<(String, Option)> = sqlx::query_as( + "SELECT path, value FROM resource WHERE workspace_id = $1 AND path = ANY($2)", + ) + .bind(w_id) + .bind(paths) + .fetch_all(db) + .await?; + + let mut candidates: Vec<(String, String)> = Vec::new(); + let mut requested: Vec<(String, Vec)> = Vec::new(); + for (path, value) in rows { + let mut owned: Vec = Vec::new(); + let mut refs: Vec = Vec::new(); + if let Some(value) = &value { + collect_var_refs(value, &mut owned); + collect_refs_with_prefixes(value, &REFERRER_PREFIXES, &mut refs); + } + candidates.extend( + owned + .into_iter() + .filter(|var_path| is_owned_linked_var(&path, var_path)) + .map(|var_path| (var_path, path.clone())), + ); + requested.push((path, refs)); + } + candidates.sort(); + candidates.dedup(); + + let mut candidate_paths: Vec = candidates + .iter() + .map(|(var_path, _)| var_path.clone()) + .collect(); + candidate_paths.dedup(); + let referenced_outside = + linked_vars_referenced_elsewhere(db, w_id, &candidate_paths, paths).await?; + Ok(LinkedVarCascade { candidates, requested, referenced_outside }) +} + +/// Deleting a resource cascades into the `$var:` variables it owns. A scoped token must not +/// use that cascade to delete variables it could not delete directly via `delete_variable` +/// (which gates on `variables:write:`), so require `variables:write` for EVERY cascaded +/// variable and fail the whole delete otherwise. /// /// No co-located-path exemption: a resource and a variable may share a path, and a /// resource-write token can create a resource over an existing standalone variable and @@ -1621,111 +1830,91 @@ async fn delete_resources_bulk( return Err(Error::PermissionDenied(msg)); } + let cascade = plan_linked_var_cascade(&db, &w_id, &request.paths).await?; + let mut tx = user_db.begin(&authed).await?; - // Capture resources for trashbin per path before bulk delete, and - // collect $var: references so we can cascade-delete the linked variables - // (matching single-resource delete semantics). - let mut linked_var_paths: Vec = Vec::new(); - for path in &request.paths { - let trash_resource: Option = sqlx::query_scalar( - "SELECT to_jsonb(t) FROM resource t WHERE path = $1 AND workspace_id = $2", - ) - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - if let Some(res_data) = trash_resource { - // Per-resource linked vars so each resource's trash entry carries - // exactly the variables that vanished with it (matching the - // single-delete shape: trash_data["linked_variables"]). - let mut this_linked: Vec = Vec::new(); - if let Some(value) = res_data.get("value") { - collect_var_refs(value, &mut this_linked); - } - this_linked.sort(); - this_linked.dedup(); - - let trash_linked_vars: Vec = if this_linked.is_empty() { - Vec::new() - } else { - let placeholders: Vec = this_linked - .iter() - .enumerate() - .map(|(i, _)| format!("${}", i + 2)) - .collect(); - let query = format!( - "SELECT to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path IN ({})", - placeholders.join(", ") - ); - let mut q = sqlx::query_scalar::<_, serde_json::Value>(&query).bind(&w_id); - for var_path in &this_linked { - q = q.bind(var_path); - } - q.fetch_all(&mut *tx).await? - }; - - let mut trash_data = serde_json::json!({"row": res_data}); - if !trash_linked_vars.is_empty() { - trash_data["linked_variables"] = serde_json::Value::Array(trash_linked_vars); - } - windmill_common::trashbin::move_to_trash( - &mut *tx, - &w_id, - "resource", - path, - trash_data, - &authed.username, - ) - .await?; - - linked_var_paths.extend(this_linked); - } - } - linked_var_paths.sort(); - linked_var_paths.dedup(); - - // A scoped token must not delete linked variables it lacks variables:write for. - check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + // Whole rows out of the delete; see delete_resource. RLS can leave a requested resource + // standing, so everything below is driven by this list rather than by `request.paths`. + let deleted: Vec<(String, serde_json::Value)> = sqlx::query_as( + "DELETE FROM resource AS t WHERE t.path = ANY($1) AND t.workspace_id = $2 + RETURNING t.path, to_jsonb(t)", + ) + .bind(&request.paths) + .bind(&w_id) + .fetch_all(&mut *tx) + .await?; + let deleted_paths: Vec = deleted.iter().map(|(path, _)| path.clone()).collect(); sqlx::query!( "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = ANY($2)", w_id, - &request.paths + &deleted_paths ) .execute(&mut *tx) .await?; - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id + let linked_var_paths = cascade.resolve(&deleted_paths); + + // A scoped token must not delete linked variables it lacks variables:write for. Erroring + // here rolls the resource deletes back with it. + check_linked_var_delete_scopes(&authed, &linked_var_paths)?; + + // Snapshot before the delete below: the trashbin entries need the rows. + let trash_linked_vars: Vec<(String, serde_json::Value)> = sqlx::query_as( + "SELECT path, to_jsonb(t) FROM variable t WHERE workspace_id = $1 AND path = ANY($2)", + ) + .bind(&w_id) + .bind(&linked_var_paths) + .fetch_all(&mut *tx) + .await?; + + let deleted_linked_variables = sqlx::query_scalar!( + "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &linked_var_paths ) .fetch_all(&mut *tx) .await?; - // Cascade-clean linked variables: delete any ws_specific 'variable' rows - // (typically auto-inserted by mark_linked_variables_ws_specific when the - // resource was ws_specific) BEFORE deleting the variable rows themselves - // — otherwise those ws_specific rows survive as orphans and a later - // variable created at the same path would inherit a stale flag. - if !linked_var_paths.is_empty() { - sqlx::query!( - "DELETE FROM ws_specific - WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", - w_id, - &linked_var_paths - ) - .execute(&mut *tx) - .await?; + // See delete_resource: ws_specific has no FK, so the rows would orphan. + sqlx::query!( + "DELETE FROM ws_specific + WHERE workspace_id = $1 AND item_kind = 'variable' AND path = ANY($2)", + w_id, + &deleted_linked_variables + ) + .execute(&mut *tx) + .await?; - sqlx::query!( - "DELETE FROM variable WHERE workspace_id = $1 AND path = ANY($2)", - w_id, - &linked_var_paths + for (path, res_data) in &deleted { + // Every cascaded variable this resource's value points at, ownership aside: restoring + // it on its own must bring back each secret it needs, and the one it borrowed from a + // sibling in the same batch is gone too. + let mut refs: Vec = Vec::new(); + if let Some(value) = res_data.get("value") { + collect_var_refs(value, &mut refs); + } + let this_linked: Vec = trash_linked_vars + .iter() + .filter(|(var_path, _)| { + refs.contains(var_path) && deleted_linked_variables.contains(var_path) + }) + .map(|(_, row)| row.clone()) + .collect(); + + let mut trash_data = serde_json::json!({"row": res_data}); + if !this_linked.is_empty() { + trash_data["linked_variables"] = serde_json::Value::Array(this_linked); + } + windmill_common::trashbin::move_to_trash( + &mut *tx, + &w_id, + "resource", + path, + trash_data, + &authed.username, ) - .execute(&mut *tx) .await?; } @@ -1740,13 +1929,30 @@ async fn delete_resources_bulk( ) .await?; + // See delete_resource: a cascaded variable gets the audit row it would have had. + for var_path in &deleted_linked_variables { + let params = cascade + .owner_of(var_path, &deleted_paths) + .map(|resource_path| HashMap::from([("via_resource", resource_path)])); + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(var_path), + params, + ) + .await?; + } + tx.commit().await?; // Wipe ALL users' drafts at these paths (and linked variables); see delete_resource. for path in &deleted_paths { delete_all_drafts_for_path(&db, &w_id, UserDraftItemKind::Resource, path).await?; } - for var_path in &linked_var_paths { + for var_path in &deleted_linked_variables { delete_all_drafts_for_path(&db, &w_id, UserDraftItemKind::Variable, var_path).await?; } @@ -2535,7 +2741,7 @@ async fn list_resource_types( ) -> JsonResult> { let rows = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ BY name", &w_id ) @@ -2560,6 +2766,352 @@ async fn list_resource_types_names( Ok(Json(rows)) } +#[derive(Serialize)] +struct ResourceTypeCount { + resource_type: String, + count: i64, +} + +/// How many resources of each type this workspace holds — how popular a type is *here*, +/// which is what the pickers rank on below the hub's own pick counts. +async fn list_resource_counts_by_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + // A count per type is aggregate, so there is no path to narrow it by: a token scoped + // to individual resources gets nothing rather than a total spanning paths it cannot + // read. Callers treat the refusal as "no local signal". + check_scopes(&authed, || "resources:read".to_string())?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| ResourceTypeCount { resource_type: r.resource_type, count: r.count }) + .collect(), + )) +} + +/// A hub read, remembered with the hub it came from: `hub_base_url` is a live instance +/// setting, so a cache ignoring it would keep serving the previous hub's answers. +struct HubCached { + hub_base_url: String, + fetched_at: std::time::Instant, + value: T, +} + +/// The index changes only when a resource type is published to the hub; picks move slowly +/// and only reorder a list. Both are read on every drawer open, hence caching at all. +const HUB_RT_INDEX_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const HUB_RT_PICKS_TTL: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// How long a *failed* index read is remembered. Short, and deliberately not the hour a +/// success is good for: this read is on the path of every picker open, so an unreachable +/// hub must not cost an outbound timeout each time, while one blip must not silence pick +/// reporting for an hour. +const HUB_RT_INDEX_FAILURE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +static HUB_RT_INDEX: LazyLock< + std::sync::RwLock>>>>, +> = LazyLock::new(|| std::sync::RwLock::new(None)); +static HUB_RT_PICKS: LazyLock>>>> = + LazyLock::new(|| std::sync::RwLock::new(None)); + +fn hub_cache_get( + cache: &std::sync::RwLock>>, + hub_base_url: &str, + ttl: std::time::Duration, +) -> Option { + let guard = cache.read().ok()?; + let entry = guard.as_ref()?; + (entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl) + .then(|| entry.value.clone()) +} + +fn hub_cache_put(cache: &std::sync::RwLock>>, hub_base_url: &str, value: T) { + if let Ok(mut guard) = cache.write() { + *guard = Some(HubCached { + hub_base_url: hub_base_url.to_string(), + fetched_at: std::time::Instant::now(), + value, + }); + } +} + +#[derive(Deserialize)] +struct HubResourceTypeEntry { + id: i64, + name: String, + /// Optional so a hub that does not send it costs only the mapping. Required, it would + /// fail the whole parse and take pick reporting — which needs just the id — with it. + #[serde(default)] + app: Option, +} + +#[derive(Clone)] +struct HubResourceType { + id: i64, + /// The integration the type belongs to. Usually the type's own name, but not always: + /// `discord_webhook` and `discord_bot_configuration` are both `discord`, and only the + /// hub knows that. Without it a workspace holding a `discord_webhook` resource looks + /// like one that has never touched Discord. + app: String, +} + +/// Reads the index cache, choosing the TTL by what is stored: a failure expires far sooner +/// than a success. `None` is a miss, `Some(None)` a remembered failure. +fn hub_index_cached(hub_base_url: &str) -> Option>> { + let guard = HUB_RT_INDEX.read().ok()?; + let entry = guard.as_ref()?; + if entry.hub_base_url != hub_base_url { + return None; + } + let ttl = if entry.value.is_some() { + HUB_RT_INDEX_TTL + } else { + HUB_RT_INDEX_FAILURE_TTL + }; + (entry.fetched_at.elapsed() < ttl).then(|| entry.value.clone()) +} + +/// What the hub knows about every published resource type, keyed by the name Windmill +/// addresses it by. `None` when the hub cannot be reached or does not answer with a list. +async fn hub_resource_types( + db: &DB, + hub_base_url: &str, +) -> Option> { + if let Some(cached) = hub_index_cached(hub_base_url) { + return cached; + } + let index = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/list"), + false, + None, + Some(db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + // Only the id and the app are kept. That listing carries every type's schema — + // around a megabyte — and neither reporting a pick nor grouping types by + // integration needs it. + Some( + response + .json::>() + .await + .ok()? + .into_iter() + .map(|rt| { + let app = rt.app.unwrap_or_else(|| rt.name.clone()); + (rt.name, HubResourceType { id: rt.id, app }) + }) + .collect::>(), + ) + } + .await; + + hub_cache_put(&HUB_RT_INDEX, hub_base_url, index.clone()); + index +} + +#[derive(Serialize)] +struct PickHubResourceTypeResult { + success: bool, +} + +/// Tells the hub a resource type was taken into a workspace, which is the counter its +/// `/resource_types/picked` ranking reads. +/// +/// Never fails the caller: a hub predating the route, an unreachable one, and a type that +/// is local-only all mean the same thing — not counted — and the request that reaches here +/// has already saved the user's resource. +/// +/// POST, and scoped as a write, because it changes state on the hub under the instance's +/// own credentials. The sibling `/type/*` routes are metadata reads that a `resources:run` +/// app-embed token may make, and both the method and this check keep such a token — which +/// is untrusted app JavaScript — from driving hub counters through us. +async fn pick_hub_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Path((_w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || "resources:write".to_string())?; + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let success = async { + let id = hub_resource_types(&db, &hub_base_url).await?.get(&name)?.id; + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/{id}/pick"), + false, + None, + Some(&db), + ) + .await + .ok()?; + Some(response.status().is_success()) + } + .await + .unwrap_or(false); + + if !success { + tracing::debug!("hub did not record a pick for resource type {name}"); + } + Ok(Json(PickHubResourceTypeResult { success })) +} + +#[derive(Deserialize, Clone)] +struct HubResourceTypePicks { + name: String, + /// The hub counts picks in a bigint, which its driver serialises as a string. + #[serde(deserialize_with = "windmill_common::more_serde::maybe_number")] + picks: i64, +} + +#[derive(Deserialize)] +struct HubPickedResourceTypes { + resource_types: Vec, +} + +#[cfg(test)] +mod hub_picks_tests { + use super::*; + + /// Two properties of the index cache that a later edit could quietly drop: it is keyed on + /// the hub it came from, and a failure is forgotten long before a success is. + #[test] + fn the_index_cache_is_keyed_on_the_hub_and_forgets_failures_sooner() { + let index = || { + Some(HashMap::from([( + "slack".to_string(), + HubResourceType { id: 1, app: "slack".to_string() }, + )])) + }; + + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", index()); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_some())); + // Switching hubs must miss rather than serve the previous hub's mapping. + assert!(hub_index_cached("https://other.example").is_none()); + + // A remembered failure reads as a hit (so the hub is not re-attempted) carrying + // nothing, and only until the shorter of the two TTLs. + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", None); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_none())); + assert!(HUB_RT_INDEX_FAILURE_TTL < HUB_RT_INDEX_TTL); + + if let Ok(mut guard) = HUB_RT_INDEX.write() { + *guard = None; + } + } + + /// The hub counts picks in a bigint, which postgres.js serialises as a string. Typing + /// the field as a plain i64 fails the whole response, and the ranking silently empties. + #[test] + fn picks_decode_from_a_string_or_a_number() { + let parsed: HubPickedResourceTypes = serde_json::from_str( + r#"{"resource_types":[{"name":"slack","picks":"42"},{"name":"github","picks":7}]}"#, + ) + .unwrap(); + assert_eq!( + parsed + .resource_types + .iter() + .map(|rt| (rt.name.as_str(), rt.picks)) + .collect::>(), + vec![("slack", 42), ("github", 7)] + ); + } +} + +/// One hub resource type as the pickers need it. +#[derive(Serialize)] +struct HubResourceTypeInfo { + name: String, + /// The integration it belongs to, so a caller can total a workspace's resources per + /// integration rather than per type. + app: String, + picks: i64, +} + +/// What the hub knows about its resource types: which integration each belongs to, and how +/// often each has been picked. +/// +/// Empty rather than an error when the hub answers neither read, so the pickers treat an +/// older or private hub as "no hub signal" and fall back to what the workspace itself uses. +/// The two reads degrade independently: a hub that lists types but has no `picked` route +/// still supplies the type-to-integration mapping, which is what decides whether a +/// workspace's resources are recognised as belonging to an integration at all. +async fn list_hub_resource_type_info( + Extension(db): Extension, +) -> JsonResult> { + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let picks = match hub_cache_get(&HUB_RT_PICKS, &hub_base_url, HUB_RT_PICKS_TTL) { + Some(picks) => picks, + None => { + let fetched = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/picked"), + false, + Some(vec![("limit", "200".to_string())]), + Some(&db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + Some( + response + .json::() + .await + .ok()? + .resource_types, + ) + } + .await + .unwrap_or_default(); + hub_cache_put(&HUB_RT_PICKS, &hub_base_url, fetched.clone()); + fetched + } + }; + + let mut picks_by_name: HashMap = + picks.into_iter().map(|rt| (rt.name, rt.picks)).collect(); + let index = hub_resource_types(&db, &hub_base_url) + .await + .unwrap_or_default(); + + let mut info: Vec = index + .into_iter() + .map(|(name, rt)| HubResourceTypeInfo { + picks: picks_by_name.remove(&name).unwrap_or(0), + name, + app: rt.app, + }) + .collect(); + // What the index did not account for is a type the picks read knows and the listing does + // not — which is what a hub answering only the second read looks like. Its own name is + // the same guess a caller makes for any type the mapping misses. + info.extend( + picks_by_name + .into_iter() + .map(|(name, picks)| HubResourceTypeInfo { app: name.clone(), name, picks }), + ); + + Ok(Json(info)) +} + async fn get_resource_type( authed: ApiAuthed, Extension(user_db): Extension, @@ -2569,7 +3121,7 @@ async fn get_resource_type( let resource_type_o = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", &name, &w_id ) @@ -2597,6 +3149,20 @@ async fn exists_resource_type( Ok(Json(exists)) } +/// Trimmed, blank as none, and held to the column's 100 characters, so an over-long name is +/// refused with a message rather than a database error. +fn normalize_display_name(name: Option<&str>) -> Result> { + let Some(name) = name.map(str::trim).filter(|n| !n.is_empty()) else { + return Ok(None); + }; + if name.chars().count() > 100 { + return Err(Error::BadRequest( + "display_name must be at most 100 characters".to_string(), + )); + } + Ok(Some(name.to_string())) +} + async fn create_resource_type( authed: ApiAuthed, Extension(db): Extension, @@ -2630,11 +3196,12 @@ async fn create_resource_type( "A fileset resource type cannot have a format_extension".to_string(), )); } + let display_name = normalize_display_name(resource_type.display_name.as_deref())?; sqlx::query!( "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", w_id, resource_type.name, resource_type.schema, @@ -2642,6 +3209,7 @@ async fn create_resource_type( authed.username, resource_type.format_extension, is_fileset, + display_name, ) .execute(&mut *tx) .await?; @@ -2803,10 +3371,48 @@ async fn update_resource_type( if let Some(is_fileset) = ns.is_fileset { sqlb.set("is_fileset", if is_fileset { "TRUE" } else { "FALSE" }); } + if let Some(format_extension) = ns.format_extension.clone() { + match format_extension { + Some(ext) => sqlb.set_str("format_extension", ext), + None => sqlb.set("format_extension", "NULL"), + }; + } + if let Some(display_name) = &ns.display_name { + match normalize_display_name(display_name.as_deref())? { + Some(name) => sqlb.set_str("display_name", name), + None => sqlb.set("display_name", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + // Creation refuses the pair outright, so an edit must too — otherwise the same + // impossible type (a set of files that is also one file) is reachable by setting + // either half on an existing row. Whichever half the request omits is read from + // the row being edited, inside this transaction and with the row locked: read + // outside it, two concurrent edits each supplying one half would both pass. + let current = sqlx::query!( + "SELECT is_fileset, format_extension FROM resource_type + WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + &name, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + let effective_is_fileset = ns + .is_fileset + .unwrap_or_else(|| current.as_ref().map(|c| c.is_fileset).unwrap_or(false)); + let effective_format_extension = match &ns.format_extension { + Some(value) => value.clone(), + None => current.and_then(|c| c.format_extension), + }; + if effective_is_fileset && effective_format_extension.is_some() { + return Err(Error::BadRequest( + "A fileset resource type cannot have a format_extension".to_string(), + )); + } + sqlx::query(&sql).execute(&mut *tx).await?; audit_log( &mut *tx, @@ -3027,8 +3633,10 @@ fn git_url_userinfo(url: &str) -> Option<&str> { git_url_userinfo_range(url).map(|r| &url[r]) } -/// Validates a git URL to prevent option injection, SSRF, and local file read. -async fn validate_git_url(url: &str) -> Result<()> { +/// Validates a git URL to prevent option injection, SSRF, and local file read. The +/// syntax and scheme checks apply to every caller; the private-host refusal only +/// where [`private_git_host_allowed`] refuses `caller`. +async fn validate_git_url(url: &str, caller: GitRemoteCaller) -> Result<()> { let url = url.trim(); if url.is_empty() { return Err(Error::BadRequest("Git URL cannot be empty".to_string())); @@ -3082,25 +3690,26 @@ async fn validate_git_url(url: &str) -> Result<()> { let host = extract_host_from_git_url(url) .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; - // CI/dev escape hatch: integration tests run their git remote (a Gitea - // container) on localhost, which the network-target checks below reject. - // Scheme and option-injection validation above still applies. - if std::env::var("ALLOW_LOCAL_GIT_REMOTES").is_ok_and(|v| v == "true" || v == "1") { + // Scheme and option-injection validation above applies to every caller. + if private_git_host_allowed(caller) { return Ok(()); } + let hint = private_git_host_hint(caller) + .map(|h| format!(" {h}")) + .unwrap_or_default(); if host == "localhost" || host.ends_with(".local") || host == "[::1]" { - return Err(Error::BadRequest( - "Git URLs targeting localhost or local network are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting localhost or local network are not allowed.{hint}" + ))); } // Check literal IP addresses if let Ok(ip) = host.parse::() { if is_private_or_reserved_ip(&ip) { - return Err(Error::BadRequest( - "Git URLs targeting private or reserved IP addresses are not allowed".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URLs targeting private or reserved IP addresses are not allowed.{hint}" + ))); } } else { // Hostname — resolve via DNS and reject if any address is private. Fail @@ -3121,9 +3730,9 @@ async fn validate_git_url(url: &str) -> Result<()> { } for addr in addrs { if is_private_or_reserved_ip(&addr.ip()) { - return Err(Error::BadRequest( - "Git URL hostname resolves to a private or reserved IP address".to_string(), - )); + return Err(Error::BadRequest(format!( + "Git URL hostname resolves to a private or reserved IP address.{hint}" + ))); } } } @@ -3225,8 +3834,24 @@ async fn get_git_commit_hash( .map_err(|e| { Error::BadRequest(format!("Invalid git repository resource format: {}", e)) })?; + let caller = if authed.is_admin { + GitRemoteCaller::AdminOrSystem + } else { + GitRemoteCaller::NonAdmin + }; git_resource.url = - resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false).await?; + resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false, caller) + .await?; + // A credential is stored under the repository it was issued for, so a + // resource repointed elsewhere finds none. Which credential can be attached + // is bounded by that; who may use it is bounded here, on the same terms as + // the installation credential above. + let plain_url = git_resource.url.clone(); + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(&db, &w_id, git_resource.url).await?; + if git_resource.url != plain_url { + require_admin(authed.is_admin, &authed.username)?; + } let identities: Vec = query .git_ssh_identity @@ -3246,7 +3871,7 @@ async fn get_git_commit_hash( let (git_ssh_cmd, filenames) = get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd, caller).await; delete_paths(&filenames).await; @@ -3350,12 +3975,17 @@ async fn get_git_ssh_cmd( const GIT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// `git` command for a remote probe, with HTTP redirects disabled. `validate_git_url` -/// only vets the host in the URL; git's default (`http.followRedirects=initial`) -/// would let a validated public remote 302 the probe onto a private or link-local -/// address that no check ever sees. Build every probe through this. +/// checks the host in the URL, never one a redirect names; git's default +/// (`http.followRedirects=initial`) would let a public remote 302 the probe of a +/// caller refused private hosts onto one. Build every probe through this. +/// +/// The transports are pinned too: an SCP-shaped remote-helper string such as +/// `ext::@host:path` passes the URL check for a caller allowed private +/// hosts, and only git's own config would stop it from running the command. fn git_probe_command() -> Command { let mut git_cmd = Command::new("git"); git_cmd.args(["-c", "http.followRedirects=false"]); + git_cmd.env("GIT_ALLOW_PROTOCOL", "http:https:ssh:git"); git_cmd } @@ -3415,8 +4045,8 @@ fn dot_git_url(url: &str) -> Option { } /// Run a remote probe, retrying against [`dot_git_url`] if the remote answered the -/// URL as given with a redirect. Extending the path keeps the retry on the host -/// `validate_git_url` already cleared, which is exactly what following the redirect +/// URL as given with a redirect. Extending the path keeps the retry on the host of +/// the URL `validate_git_url` checked, which is exactly what following the redirect /// would not guarantee. `build` must produce the probe for the URL it is handed. /// /// A retry that also fails reports the *original* failure, so the caller's message @@ -3556,6 +4186,7 @@ async fn resolve_azure_devops_url( w_id: &str, url: &str, allow_cache: bool, + caller: GitRemoteCaller, ) -> Result { // Trim first: the http(s) gates the callers apply trim too, so a stored URL with // leading whitespace must not reach the scheme check here as a non-http one. @@ -3568,7 +4199,7 @@ async fn resolve_azure_devops_url( // cost a live credential (nor cache one), and whoever can edit the URL would // otherwise drive a token mint per poll tick. let probe_url = url.replace(placeholder, "windmill"); - validate_git_url(&probe_url).await?; + validate_git_url(&probe_url, caller).await?; // The background poller reads the referenced resource under the system identity, // which bypasses RLS. Confining the destination is what keeps that from becoming an @@ -3768,9 +4399,10 @@ fn git_sync_system_dba(db: &DB) -> DbWithOptAuthed<'static, ApiAuthed> { async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, + caller: GitRemoteCaller, ) -> Result { // Validate URL and branch to prevent option injection and SSRF attacks - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, caller).await?; let ref_spec = git_resource .branch @@ -3903,19 +4535,30 @@ pub async fn get_git_repo_head_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } + git_resource.url = resolve_azure_devops_url( + &git_sync_system_dba(db), + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; + // A repo whose credential Windmill holds carries none in its URL, so the + // poller has to attach it here or every probe would be unauthenticated. git_resource.url = - resolve_azure_devops_url(&git_sync_system_dba(db), w_id, &git_resource.url, true).await?; + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { let branch = branch.to_string(); - let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + let sha = get_repo_latest_commit_hash(&git_resource, None, GitRemoteCaller::AdminOrSystem) + .await?; return Ok(Some((branch, sha))); } // No explicit branch: resolve the remote's default-branch NAME along with // its head in one call. Fork sync needs the concrete name to scope // `wm-fork//*`, so a bare "HEAD" ref would silently disable it. - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; let output = run_git_probe_for_url(&git_resource.url, "ls-remote --symref HEAD", |url| { let mut git_cmd = git_probe_command(); git_cmd.args(["ls-remote", "--symref", url, "HEAD"]); @@ -4010,8 +4653,20 @@ pub async fn get_git_repo_fork_heads_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = resolve_azure_devops_url(&dba, w_id, &git_resource.url, true).await?; - validate_git_url(&git_resource.url).await?; + git_resource.url = resolve_azure_devops_url( + &dba, + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; + // Same reason as the head probe above: a repository whose credential Windmill + // holds carries none in its URL, and listing the fork branches is the half of + // polling that would otherwise go out unauthenticated. + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; validate_git_ref(base_branch)?; for r in extra_refs { @@ -4397,46 +5052,52 @@ mod tests { )); } + // A caller let through to private hosts must still hit the scheme check. #[tokio::test] async fn test_validate_git_url_blocks_file_scheme() { - let result = validate_git_url("file:///etc/passwd").await; + let result = validate_git_url("file:///etc/passwd", GitRemoteCaller::AdminOrSystem).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("https://")); } #[tokio::test] async fn test_validate_git_url_blocks_private_ips() { - assert!(validate_git_url("http://127.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://169.254.169.254/latest/meta-data/") - .await - .is_err()); - assert!(validate_git_url("http://10.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://172.16.0.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("http://192.168.1.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("git://0.0.0.0/repo.git").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1/repo.git").await.is_err()); + assert!(v("http://169.254.169.254/latest/meta-data/").await.is_err()); + let err = v("http://10.0.0.1/repo.git").await.unwrap_err(); + assert!(err.to_string().contains("ALLOW_LOCAL_GIT_REMOTES"), "{err}"); + assert!(v("http://172.16.0.1/repo.git").await.is_err()); + assert!(v("http://192.168.1.1/repo.git").await.is_err()); + assert!(v("git://0.0.0.0/repo.git").await.is_err()); // IPv6 loopback, unique-local, and link-local literals - assert!(validate_git_url("git://[::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fd00::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fe80::1]/repo.git").await.is_err()); + assert!(v("git://[::1]/repo.git").await.is_err()); + assert!(v("git://[fd00::1]/repo.git").await.is_err()); + assert!(v("git://[fe80::1]/repo.git").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_lets_admins_reach_private_hosts() { + assert!( + validate_git_url("http://10.0.0.1/repo.git", GitRemoteCaller::AdminOrSystem) + .await + .is_ok() + ); } #[tokio::test] async fn test_validate_git_url_blocks_localhost() { - assert!(validate_git_url("http://localhost/repo.git").await.is_err()); - assert!(validate_git_url("http://myhost.local/repo.git") - .await - .is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://localhost/repo.git").await.is_err()); + assert!(v("http://myhost.local/repo.git").await.is_err()); } #[tokio::test] async fn test_validate_git_url_blocks_local_paths() { - assert!(validate_git_url("/etc/passwd").await.is_err()); - assert!(validate_git_url("../relative/path").await.is_err()); - assert!(validate_git_url("./local/repo").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("/etc/passwd").await.is_err()); + assert!(v("../relative/path").await.is_err()); + assert!(v("./local/repo").await.is_err()); } /// Minimal loopback HTTP server: replies to every request with `response` and @@ -4566,7 +5227,11 @@ mod tests { async fn test_validate_git_url_fails_closed_on_unresolvable_host() { // `.invalid` never resolves (RFC 6761). The private-IP check is only // meaningful if a failed lookup rejects instead of falling through. - let result = validate_git_url("https://this-host-does-not-exist.invalid/repo.git").await; + let result = validate_git_url( + "https://this-host-does-not-exist.invalid/repo.git", + GitRemoteCaller::NonAdmin, + ) + .await; assert!( result.is_err(), "an unresolvable host was allowed — does this resolver synthesize records for NXDOMAIN?" @@ -4577,21 +5242,33 @@ mod tests { #[tokio::test] async fn test_validate_git_url_allows_valid_urls() { // Needs DNS: validation fails closed on a host it cannot resolve. - assert!(validate_git_url("https://github.com/user/repo.git") + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("https://github.com/user/repo.git").await.is_ok()); + assert!(v("git@github.com:user/repo.git").await.is_ok()); + assert!(v("ssh://git@github.com/user/repo.git").await.is_ok()); + } + + #[tokio::test] + async fn test_git_probe_refuses_remote_helpers() { + // A caller allowed private hosts skips the DNS step that would reject this + // SCP-shaped string, so the transport pin is what keeps git from running it. + let output = git_probe_command() + .args(["ls-remote", "testhelper::x@127.0.0.1:repo"]) + .output() .await - .is_ok()); - assert!(validate_git_url("git@github.com:user/repo.git") - .await - .is_ok()); - assert!(validate_git_url("ssh://git@github.com/user/repo.git") - .await - .is_ok()); + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("transport 'testhelper' not allowed"), + "{stderr}" + ); } #[tokio::test] async fn test_validate_git_url_blocks_option_injection() { - assert!(validate_git_url("-evil").await.is_err()); - assert!(validate_git_url("--upload-pack=evil").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("-evil").await.is_err()); + assert!(v("--upload-pack=evil").await.is_err()); } #[test] @@ -4687,24 +5364,21 @@ mod tests { // GHSA-p5cj-8cfh-mjv6: a loopback authority must stay blocked, and the // fragment/query `@public-host` bypasses of #8600 must be rejected so the // host git dials can never diverge from the validated host. - assert!(validate_git_url("http://127.0.0.1:40173/repo.git") - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - // A legitimate public repo URL still validates. + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1:40173/repo.git").await.is_err()); assert!( - validate_git_url("https://github.com/windmill-labs/windmill.git") + v("http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git") .await - .is_ok() + .is_err() ); + assert!( + v("http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git") + .await + .is_err() + ); + // A legitimate public repo URL still validates. + assert!(v("https://github.com/windmill-labs/windmill.git") + .await + .is_ok()); } } diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 3766a6050d..773616a807 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -158,6 +158,7 @@ pub struct RunJob { pub payload: JobPayload, pub args: serde_json::Map, pub scheduled_for_o: Option>, + pub username: String, pub email: String, pub job_id: Option, pub workspace_id: String, @@ -169,6 +170,7 @@ impl From for RunJob { payload, args: Default::default(), scheduled_for_o: None, + username: "test-user".to_string(), email: "test@windmill.dev".to_string(), job_id: None, workspace_id: "test-workspace".to_string(), @@ -190,7 +192,11 @@ impl RunJob { self } - pub fn email(mut self, email: impl Into) -> Self { + /// Run as this workspace member. Both halves together, because the job's identity is the + /// principal: an address paired with another member's username is re-resolved at push to + /// the address that username holds. + pub fn as_user(mut self, username: impl Into, email: impl Into) -> Self { + self.username = username.into(); self.email = email.into(); self } @@ -206,7 +212,7 @@ impl RunJob { } pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o, email, job_id, workspace_id } = self; + let RunJob { payload, args, scheduled_for_o, username, email, job_id, workspace_id } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -219,9 +225,9 @@ impl RunJob { &workspace_id, payload, windmill_queue::PushArgs::from(&hm_args), - /* user */ "test-user", + /* user */ &username, /* email */ &email, - /* permissioned_as */ "u/test-user".to_string(), + /* permissioned_as */ format!("u/{username}"), /* token_prefix */ None, /* audit_end_user */ None, scheduled_for_o, diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 14e1b9a4d2..01ddfd4a81 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -9,7 +9,7 @@ use sqlx::PgConnection; use std::collections::HashSet; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::global_settings::HTTP_ROUTE_WORKSPACED_ROUTE; +use windmill_common::global_settings::{validate_allowed_origins, HTTP_ROUTE_WORKSPACED_ROUTE}; use windmill_common::{ db::UserDB, error::{Error, Result}, @@ -189,6 +189,7 @@ pub async fn insert_new_trigger_into_db( authentication_resource_path, wrap_body, raw_string, + allowed_origins, script_path, summary, description, @@ -207,7 +208,7 @@ pub async fn insert_new_trigger_into_db( retry ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, now(), $21, $22, $23, $24 ) "#, w_id, @@ -218,6 +219,7 @@ pub async fn insert_new_trigger_into_db( trigger.config.authentication_resource_path, trigger.config.wrap_body.unwrap_or(false), trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.base.script_path, trigger.config.summary, trigger.config.description, @@ -268,6 +270,15 @@ pub async fn create_many_http_triggers( format!("http_triggers:write:{}", &new_http_trigger.base.path) })?; + // This route inserts directly, bypassing the shared create handler. + // `error_wrapper` would turn the rejection into a 500. + new_http_trigger.error_handling.validate().map_err(|err| { + Error::BadRequest(format!( + "Error occurred for HTTP route at route path: {}, error: {}", + new_http_trigger.config.route_path, err + )) + })?; + handler .validate_new(&db, &w_id, &new_http_trigger.config) .await @@ -435,6 +446,7 @@ impl TriggerCrud for HttpTrigger { "workspaced_route", "wrap_body", "raw_string", + "allowed_origins", ]; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject { @@ -465,6 +477,8 @@ impl TriggerCrud for HttpTrigger { validate_authentication_method(new.authentication_method, new.raw_string)?; + validate_allowed_origins(new.allowed_origins.as_deref().unwrap_or_default())?; + Ok(()) } @@ -483,6 +497,8 @@ impl TriggerCrud for HttpTrigger { validate_authentication_method(edit.authentication_method, edit.raw_string)?; + validate_allowed_origins(edit.allowed_origins.as_deref().unwrap_or_default())?; + Ok(()) } @@ -545,33 +561,35 @@ impl TriggerCrud for HttpTrigger { workspaced_route = $3, wrap_body = $4, raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - mode = $10, - http_method = $11, - static_asset_config = $12, - edited_by = $13, - permissioned_as = $14, - request_type = $15, - authentication_method = $16, - summary = $17, - description = $18, + allowed_origins = $6, + authentication_resource_path = $7, + script_path = $8, + path = $9, + is_flow = $10, + mode = $11, + http_method = $12, + static_asset_config = $13, + edited_by = $14, + permissioned_as = $15, + request_type = $16, + authentication_method = $17, + summary = $18, + description = $19, edited_at = now(), - is_static_website = $19, - error_handler_path = $20, - error_handler_args = $21, - retry = $22 + is_static_website = $20, + error_handler_path = $21, + error_handler_args = $22, + retry = $23 WHERE - workspace_id = $23 AND - path = $24 + workspace_id = $24 AND + path = $25 "#, route_path, &route_path_key, Some(effective_workspaced), - trigger.config.wrap_body, - trigger.config.raw_string, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, @@ -604,30 +622,32 @@ impl TriggerCrud for HttpTrigger { SET wrap_body = $1, raw_string = $2, - authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - mode = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - permissioned_as = $11, - request_type = $12, - authentication_method = $13, - summary = $14, - description = $15, + allowed_origins = $3, + authentication_resource_path = $4, + script_path = $5, + path = $6, + is_flow = $7, + mode = $8, + http_method = $9, + static_asset_config = $10, + edited_by = $11, + permissioned_as = $12, + request_type = $13, + authentication_method = $14, + summary = $15, + description = $16, edited_at = now(), - is_static_website = $16, - error_handler_path = $17, - error_handler_args = $18, - retry = $19 + is_static_website = $17, + error_handler_path = $18, + error_handler_args = $19, + retry = $20 WHERE - workspace_id = $20 AND - path = $21 + workspace_id = $21 AND + path = $22 "#, - trigger.config.wrap_body, - trigger.config.raw_string, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), + trigger.config.allowed_origins.as_deref(), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs index b78276c28d..40bab102a5 100644 --- a/backend/windmill-trigger-http/src/lib.rs +++ b/backend/windmill-trigger-http/src/lib.rs @@ -8,7 +8,7 @@ use tokio::sync::{RwLock, RwLockReadGuard}; use windmill_common::{ error::{Error, Result}, flows::Retry, - global_settings::HTTP_ROUTE_WORKSPACED_ROUTE, + global_settings::{allows_any_origin, HTTP_ROUTE_WORKSPACED_ROUTE}, utils::ExpiringCacheEntry, worker::CLOUD_HOSTED, DB, @@ -51,6 +51,7 @@ pub struct TriggerRoute { pub workspaced_route: bool, pub wrap_body: bool, pub raw_string: bool, + pub allowed_origins: Option>, pub error_handler_path: Option, pub error_handler_args: Option>>, pub retry: Option>, @@ -127,6 +128,7 @@ pub struct HttpConfig { pub workspaced_route: bool, pub wrap_body: bool, pub raw_string: bool, + pub allowed_origins: Option>, } #[derive(Debug, Clone, Serialize)] @@ -144,6 +146,7 @@ pub struct HttpConfigRequest { pub workspaced_route: Option, pub wrap_body: Option, pub raw_string: Option, + pub allowed_origins: Option>, } #[derive(Deserialize)] @@ -162,6 +165,7 @@ struct HttpConfigRequestHelper { workspaced_route: Option, wrap_body: Option, raw_string: Option, + allowed_origins: Option>, } impl<'de> Deserialize<'de> for HttpConfigRequest { @@ -197,6 +201,7 @@ impl<'de> Deserialize<'de> for HttpConfigRequest { workspaced_route: helper.workspaced_route, wrap_body: helper.wrap_body, raw_string: helper.raw_string, + allowed_origins: helper.allowed_origins, }) } } @@ -216,6 +221,50 @@ pub struct RouteExists { pub workspaced_route: Option, } +/// The allowlist that governs a route: its own when it has one, otherwise the +/// instance-wide default. `None` means nothing is configured at either level, so +/// the route keeps the historical permissive behaviour. +/// +/// A list containing `*` is treated as no restriction, which is how a route opts +/// out of a stricter instance default. +pub fn effective_allowed_origins<'a>( + route_allowed_origins: Option<&'a [String]>, + instance_default: &'a [String], +) -> Option<&'a [String]> { + // An empty list is not a configuration. It reads exactly as never having set + // one, so such a route still inherits the instance default rather than + // skipping it, which is what would make `[]` more permissive than `NULL`. + match route_allowed_origins.filter(|list| !list.is_empty()) { + // `*` is the opt-out, including out of a stricter instance default. + Some(list) if allows_any_origin(list) => None, + Some(list) => Some(list), + None => (!instance_default.is_empty() && !allows_any_origin(instance_default)) + .then_some(instance_default), + } +} + +/// Resolve the `Access-Control-Allow-Origin` value for a request, or `None` to +/// omit the header so the browser blocks the read. +/// +/// The request's `Origin` is echoed back only on a match against the allowlist. +/// Reflecting it unchecked is the classic way this feature turns into no +/// restriction at all. +/// +/// The comparison ignores ASCII case because a browser lowercases the scheme and +/// host it sends, so a configured `https://App.Example.com` would otherwise name +/// a real origin and still match nothing. +pub fn match_origin( + allowed_origins: &[String], + origin: Option<&http::HeaderValue>, +) -> Option { + let origin = origin?; + let origin_str = origin.to_str().ok()?; + allowed_origins + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(origin_str)) + .then(|| origin.clone()) +} + pub fn validate_authentication_method( authentication_method: AuthenticationMethod, raw_string: Option, @@ -276,6 +325,7 @@ pub async fn refresh_routers( static_asset_config AS "static_asset_config: _", wrap_body, raw_string, + allowed_origins, workspaced_route, is_static_website, error_handler_path, @@ -384,6 +434,10 @@ pub struct HttpTrigger; #[cfg(test)] mod tests { use super::*; + // Not used by the lib itself, only exercised here. + use windmill_common::global_settings::{ + validate_allowed_origins, MAX_ALLOWED_ORIGINS, MAX_ALLOWED_ORIGIN_LEN, + }; #[test] fn test_request_type_backward_compatibility() { @@ -578,6 +632,168 @@ mod tests { assert!(validate_authentication_method(AuthenticationMethod::Signature, None).is_ok()); } + // --- CORS allowed origins --- + + fn origin(value: &str) -> http::HeaderValue { + http::HeaderValue::from_str(value).unwrap() + } + + #[test] + fn test_match_origin_exact_match_echoes_request_origin() { + let allowed = vec!["https://a.com".to_string(), "https://b.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://b.com"))), + Some(origin("https://b.com")) + ); + } + + #[test] + fn test_match_origin_ignores_case() { + let allowed = vec!["https://App.Example.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://app.example.com"))), + Some(origin("https://app.example.com")) + ); + } + + #[test] + fn test_match_origin_no_match_omits_header() { + let allowed = vec!["https://a.com".to_string()]; + assert_eq!( + match_origin(&allowed, Some(&origin("https://evil.com"))), + None + ); + // A prefix of an allowed origin must not match: https://a.com.evil.com + // is a different site entirely. + assert_eq!( + match_origin(&allowed, Some(&origin("https://a.com.evil.com"))), + None + ); + } + + #[test] + fn test_wildcard_entry_means_unrestricted() { + // `*` is handled before matching: it means "no restriction", which is + // how a route opts out of a stricter instance default. + assert!(allows_any_origin(&["*".to_string()])); + assert!(allows_any_origin(&[ + "https://a.com".to_string(), + "*".to_string() + ])); + assert!(!allows_any_origin(&["https://a.com".to_string()])); + assert_eq!( + effective_allowed_origins(Some(&["*".to_string()]), &[]), + None + ); + } + + #[test] + fn test_effective_allowed_origins_prefers_the_route() { + let route = ["https://a.com".to_string()]; + let default = ["https://default.com".to_string()]; + assert_eq!( + effective_allowed_origins(Some(&route), &default), + Some(&route[..]) + ); + // No route list: the instance default applies. + assert_eq!( + effective_allowed_origins(None, &default), + Some(&default[..]) + ); + // No route list and no instance default: nothing is restricted, so the + // historical permissive behaviour is kept. + assert_eq!(effective_allowed_origins(None, &[]), None); + // A route opting out with `*` escapes a stricter instance default. + assert_eq!( + effective_allowed_origins(Some(&["*".to_string()]), &default), + None + ); + // An empty route list is not a configuration: it resolves exactly as + // `NULL` does, so it inherits the instance default rather than skipping + // it and becoming more permissive than an unset one. + assert_eq!( + effective_allowed_origins(Some(&[]), &default), + Some(&default[..]) + ); + assert_eq!(effective_allowed_origins(Some(&[]), &[]), None); + } + + #[test] + fn test_match_origin_missing_origin_header_omits_header() { + let allowed = vec!["https://a.com".to_string()]; + assert_eq!(match_origin(&allowed, None), None); + } + + #[test] + fn test_validate_allowed_origins_accepts_anything_comparable() { + // A shape that cannot match simply matches nothing, so it is the + // editor's job to warn and not this one's to refuse. What is refused is + // narrower: `null`, values that are not header-comparable, entries that + // cannot round-trip the editor's comma-separated field, and lists past + // the size a request can afford to scan. + let allowed = vec![ + "https://app.example.com".to_string(), + "http://localhost:3000".to_string(), + "http://[::1]:8080".to_string(), + "chrome-extension://mhjfbmdgcfjbbpaeojofohoefgiehjai".to_string(), + // Never matches, but that is the caller's problem, not an error. + "https://app.example.com/".to_string(), + "https://app.example.com:99999".to_string(), + "not-an-origin".to_string(), + "*".to_string(), + ]; + assert!(validate_allowed_origins(&allowed).is_ok()); + assert!(validate_allowed_origins(&[]).is_ok()); + } + + #[test] + fn test_parse_allowed_origins_setting_rejects_empty_array_entries() { + use windmill_common::global_settings::parse_allowed_origins_setting; + // A trailing separator in the string form is a typing artifact and is + // dropped; an empty array entry is something the caller wrote, so it + // must reach validation rather than be filtered away into an empty + // (and therefore unrestricted) default. + assert!(parse_allowed_origins_setting(Some(&serde_json::json!("https://a.com,"))).is_ok()); + assert!(parse_allowed_origins_setting(Some(&serde_json::json!([""]))).is_err()); + assert!( + parse_allowed_origins_setting(Some(&serde_json::json!(["https://a.com", ""]))).is_err() + ); + } + + #[test] + fn test_validate_allowed_origins_bounds_the_list() { + // An allowlist is scanned on every request to a restricted route, the + // unauthenticated preflight included, so its size is a cost anyone can + // trigger. + let too_many = vec!["https://a.com".to_string(); MAX_ALLOWED_ORIGINS + 1]; + assert!(validate_allowed_origins(&too_many).is_err()); + assert!(validate_allowed_origins(&too_many[..MAX_ALLOWED_ORIGINS]).is_ok()); + let too_long = format!("https://{}.com", "a".repeat(MAX_ALLOWED_ORIGIN_LEN)); + assert!(validate_allowed_origins(&[too_long]).is_err()); + } + + #[test] + fn test_validate_allowed_origins_rejects_null_and_uncomparable() { + for invalid in [ + // Every sandboxed iframe sends `Origin: null`, so allowing it would + // grant access to any page that can open one. + "null", + "NULL", // Cannot be the string an Origin header is compared against. + "https://a b.com", + "https://app.example.com ", + "https://exämple.com", + // The editor edits the list as one comma-separated field, so an + // entry carrying a comma would come back as two and widen the list. + "https://a.com,https://b.com", + "", + ] { + assert!( + validate_allowed_origins(&[invalid.to_string()]).is_err(), + "expected {invalid} to be rejected" + ); + } + } + // --- Route path regex --- #[test] diff --git a/backend/windmill-trigger-postgres/src/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs index 92dd2e1e19..4294645b89 100644 --- a/backend/windmill-trigger-postgres/src/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -21,11 +21,12 @@ use windmill_common::{ use windmill_git_sync::DeployedObject; use windmill_api_auth::{check_scopes, ApiAuthed}; -use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData, TriggerMode}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, - create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, + create_pg_publication, drop_publication, ensure_not_under_roles, generate_random_string, + get_default_pg_connection, mapper::{Mapper, MappingInfo}, PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PostgresTrigger, PublicationData, Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, @@ -64,6 +65,29 @@ impl TriggerCrud for PostgresTrigger { DeployedObject::PostgresTrigger { path, parent_path } } + async fn validate_config( + &self, + db: &DB, + config: &Self::TriggerConfigRequest, + workspace_id: &str, + ) -> Result<()> { + ensure_not_under_roles(db, workspace_id, &config.postgres_resource_path).await + } + + async fn authorize_set_trigger_mode( + &self, + _authed: &ApiAuthed, + tx: &mut PgConnection, + _workspace_id: &str, + _path: &str, + mode: &TriggerMode, + ) -> Result<()> { + if *mode != TriggerMode::Disabled { + windmill_common::datatable_roles::lock_datatable_streams(tx, false).await?; + } + Ok(()) + } + async fn create_trigger( &self, db: &DB, @@ -72,6 +96,7 @@ impl TriggerCrud for PostgresTrigger { w_id: &str, trigger: TriggerData, ) -> Result<()> { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; let resolved_edited_by = trigger.base.resolve_edited_by(authed); let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); let Self::TriggerConfigRequest { @@ -161,6 +186,7 @@ impl TriggerCrud for PostgresTrigger { path: &str, trigger: TriggerData, ) -> Result<()> { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; let resolved_edited_by = trigger.base.resolve_edited_by(authed); let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); let Self::TriggerConfigRequest { diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs index 571aac0a66..b92860662d 100644 --- a/backend/windmill-trigger-postgres/src/lib.rs +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -374,6 +374,35 @@ pub async fn get_raw_postgres_connection( Ok(client) } +/// A replication stream reads every row of every table whatever the data table's roles grant, so +/// the two don't mix: a data table under roles takes no triggers or captures, and roles cannot be +/// turned on while one is enabled on it. +/// +/// Authorization: checks nothing, and its refusal says whether `w_id`'s data table is under roles. +/// Callers MUST have established that the caller may manage triggers in `w_id` first. +pub(crate) async fn ensure_not_under_roles( + db: &DB, + w_id: &str, + postgres_resource_path: &str, +) -> Result<()> { + let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") else { + return Ok(()); + }; + if windmill_common::workspaces::resolve_governing_datatable(db, w_id, datatable_name) + .await? + .datatable + .permissions + .is_some() + { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' is under roles, and a Postgres trigger or capture \ + cannot read one: a replication stream sees every row whatever the roles grant. \ + Turn its roles off to stream it." + ))); + } + Ok(()) +} + pub async fn resolve_postgres_resource( authed: &ApiAuthed, user_db: Option, @@ -382,6 +411,7 @@ pub async fn resolve_postgres_resource( w_id: &str, ) -> Result { if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") { + ensure_not_under_roles(db, w_id, postgres_resource_path).await?; // Trigger connections (publication/slot management + logical replication) run // as the dedicated replication user on custom-instance databases. let resource_value = diff --git a/backend/windmill-trigger-postgres/src/listener.rs b/backend/windmill-trigger-postgres/src/listener.rs index 14d348b5f2..876b75661e 100644 --- a/backend/windmill-trigger-postgres/src/listener.rs +++ b/backend/windmill-trigger-postgres/src/listener.rs @@ -20,7 +20,8 @@ use windmill_common::{ use windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}; use super::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, + drop_publication, ensure_not_under_roles, get_default_pg_connection, + get_raw_postgres_connection, handler::drop_logical_replication_slot, relation::RelationConverter, replication_message::{ @@ -135,8 +136,8 @@ impl PostgresSimpleClient { /// Resolves the Postgres resource, validates that the configured publication and /// replication slot still exist, and opens a fresh logical replication stream. /// -/// Returns `Error::BadConfig` when the publication or slot is missing (an -/// unrecoverable misconfiguration). Any other error is treated as transient +/// Returns `Error::BadConfig` when the publication or slot is missing, or the +/// data table is under roles (unrecoverable misconfigurations). Any other error is treated as transient /// (connection refused, network interruption, ...) and is retried by the caller. /// The resource is re-resolved on every call so credential rotations are picked /// up across reconnections. @@ -149,6 +150,14 @@ async fn connect_logical_replication_stream( let PostgresConfig { postgres_resource_path, publication_name, replication_slot_name, .. } = trigger_config; + // Retrying cannot lift roles, so this disables the trigger like a missing slot does. + ensure_not_under_roles(db, workspace_id, postgres_resource_path) + .await + .map_err(|e| match e { + Error::BadRequest(msg) => Error::BadConfig(msg), + e => e, + })?; + let database = resolve_postgres_resource( authed, Some(UserDB::new(db.clone())), diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 28898489f4..66d6770354 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -18,8 +18,9 @@ use windmill_common::{ error::{Error, JsonResult, Result}, trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, - overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, + WithDraftQuery, }, utils::{paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, @@ -96,14 +97,14 @@ pub trait TriggerCrud: Send + Sync + 'static { const DEPLOYMENT_NAME: &'static str; const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; const IS_ALLOWED_ON_CLOUD: bool; - /// Whether enabling this trigger in a fork while the parent has the same - /// path enabled is a real conflict (shared upstream resource). True for + /// Whether enabling this trigger in a fork while an ancestor workspace has + /// the same path is a real conflict (shared upstream resource). True for /// listener-based kinds where two consumers compete (Kafka group, PG slot, /// SQS queue, etc.) and for Websocket where both subscribers fire on every /// broadcast. False for kinds whose upstream identifier is implicitly /// workspace-scoped at runtime (HTTP routes, Email local_part — clones for /// the non-workspaced sub-case are filtered out, so any cloned row is - /// already collision-free vs. the parent). + /// already collision-free vs. its ancestors). const FORK_CONFLICT_ON_ENABLE: bool = true; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject; @@ -554,6 +555,8 @@ async fn create_trigger( ))); } + new_trigger.error_handling.validate()?; + handler .validate_new(&db, &workspace_id, &new_trigger.config) .await?; @@ -815,6 +818,8 @@ async fn update_trigger( ) })?; + edit_trigger.error_handling.validate()?; + handler .validate_edit(&db, &workspace_id, &edit_trigger.config, path) .await?; @@ -986,6 +991,18 @@ async fn delete_trigger( .await?; if !deleted { + drop(tx); + if delete_draft_only_for_path( + &db, + &workspace_id, + T::user_draft_item_kind(), + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only trigger '{}' deleted", path)); + } return Err(Error::NotFound(format!( "Trigger not found at path: {}", path @@ -1078,54 +1095,14 @@ async fn exists_trigger( #[derive(serde::Deserialize)] struct SetTriggerModePayload { mode: TriggerMode, - /// When true, bypass the parent-state warning that would otherwise reject - /// enabling a trigger that's already enabled in the parent workspace. - /// The frontend sets this after the user confirms the duplicate-execution - /// dialog. See windmill-trigger/src/handler.rs::set_trigger_mode for the - /// full check. + /// When true, bypass the fork-conflict warning that would otherwise reject + /// enabling a trigger an ancestor workspace also has at this path. The + /// frontend sets this after the user confirms the duplicate-execution + /// dialog. See `set_trigger_mode` for the full check. #[serde(default)] force: bool, } -/// Returns the parent workspace id when this workspace is a fork *and* the -/// parent has a row at the same trigger path. Used to gate enabling a trigger -/// in a fork behind an explicit `force=true` confirmation: the fork's row was -/// cloned from the parent, so its upstream identifier (Kafka group, PG slot, -/// SQS queue URL, etc.) is shared by construction. The risk is independent of -/// the parent's current `mode`: if the parent is enabled, the two listeners -/// compete; if it's disabled, the fork can destructively take over shared -/// state (e.g. advance the PG WAL, claim an MQTT client_id) before the parent -/// re-enables. Either way, the user should be asked to confirm. -async fn parent_has_trigger( - tx: &mut PgConnection, - table_name: &str, - workspace_id: &str, - path: &str, -) -> Result> { - let parent: Option = - sqlx::query_scalar("SELECT parent_workspace_id FROM workspace WHERE id = $1") - .bind(workspace_id) - .fetch_optional(&mut *tx) - .await? - .flatten(); - let Some(parent_id) = parent else { - return Ok(None); - }; - let exists: Option = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - table_name - )) - .bind(&parent_id) - .bind(path) - .fetch_one(&mut *tx) - .await?; - Ok(if exists == Some(true) { - Some(parent_id) - } else { - None - }) -} - async fn set_trigger_mode( Extension(handler): Extension>, authed: ApiAuthed, @@ -1140,22 +1117,28 @@ async fn set_trigger_mode( let mut tx = user_db.begin(&authed).await?; // Block transitioning a trigger in a fork to any mode that attaches a - // listener (Enabled or Suspended) when the parent has the same path, + // listener (Enabled or Suspended) when an ancestor has the same path, // unless the caller passes force=true. Suspended still keeps the // listener attached — it just stops auto-running queued jobs — so a // suspended fork would still split Kafka events / share a PG slot - // with the parent. The cloned upstream identifier is shared by - // construction; the risk is independent of the parent's current mode. - // Skipped for kinds where the upstream identifier is already - // workspace-scoped at runtime (HTTP, Email). + // with the ancestor. The risk is independent of the ancestor's current + // mode: enabled, the two listeners compete; disabled, the fork can + // destructively take over shared state (advance the PG WAL, claim an + // MQTT client_id) before it re-enables. Skipped for kinds where the + // upstream identifier is already workspace-scoped at runtime (HTTP, Email). if T::FORK_CONFLICT_ON_ENABLE && payload.mode != TriggerMode::Disabled && !payload.force { - if let Some(parent_id) = - parent_has_trigger(&mut *tx, T::TABLE_NAME, &workspace_id, path).await? + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, + T::TABLE_NAME, + &workspace_id, + path, + ) + .await? { return Err(Error::BadRequest(format!( "fork-conflict:{}:{}", T::TRIGGER_TYPE, - parent_id + ancestor_id ))); } } diff --git a/backend/windmill-trigger/src/types.rs b/backend/windmill-trigger/src/types.rs index dcf9a136be..2c1b854396 100644 --- a/backend/windmill-trigger/src/types.rs +++ b/backend/windmill-trigger/src/types.rs @@ -84,6 +84,31 @@ pub struct TriggerErrorHandling { pub retry: Option>, } +impl TriggerErrorHandling { + /// Schedule and workspace error handlers encode script-vs-flow as a + /// `script/`/`flow/` prefix; a trigger's handler is always a script, so a + /// prefixed path here would be looked up verbatim as a script name and fail + /// only once the trigger errors, which is when the handler is needed. + pub fn validate(&self) -> windmill_common::error::Result<()> { + let Some(path) = self.error_handler_path.as_deref() else { + return Ok(()); + }; + if let Some(bare) = path.strip_prefix("script/") { + return Err(windmill_common::error::Error::BadRequest(format!( + "error_handler_path is a plain script path, not the prefixed form a schedule \ + error handler takes: got '{path}', use '{bare}'" + ))); + } + if path.starts_with("flow/") { + return Err(windmill_common::error::Error::BadRequest(format!( + "error_handler_path must be a script: a trigger error handler cannot be a flow \ + (got '{path}')" + ))); + } + Ok(()) + } +} + #[derive(Serialize, Deserialize, Clone)] pub struct Trigger where @@ -253,6 +278,43 @@ mod tests { use super::*; use serde_json::json; + // --- TriggerErrorHandling::validate --- + + fn error_handling(path: Option<&str>) -> TriggerErrorHandling { + TriggerErrorHandling { + error_handler_path: path.map(str::to_string), + error_handler_args: None, + retry: None, + } + } + + #[test] + fn test_error_handler_path_accepts_bare_and_hub_paths() { + for path in [None, Some("f/team/handler"), Some("u/admin/handler")] { + assert!(error_handling(path).validate().is_ok(), "{path:?}"); + } + assert!(error_handling(Some("hub/13953/windmill/handler")) + .validate() + .is_ok()); + } + + #[test] + fn test_error_handler_path_rejects_prefixed_paths() { + // The rejection names the bare path to use, so the caller can fix it + // without knowing which of the two conventions a trigger follows. + let err = error_handling(Some("script/f/team/handler")) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("f/team/handler"), "{err}"); + + let err = error_handling(Some("flow/f/team/handler")) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("cannot be a flow"), "{err}"); + } + // --- TriggerMode serde --- #[test] diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 02feeb421a..a2fc7fd58d 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -98,6 +98,11 @@ pub struct ListableFlow { pub deployment_msg: Option, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + /// Projected from the flow value so a list can mark a flow that opens as a + /// chat without fetching every flow's value. + #[sqlx(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_input_enabled: Option, /// True when the authed user has a draft for this flow (draft-only or layered /// over the deployed row). See ListableScript in scripts.rs. #[serde(default)] @@ -538,6 +543,20 @@ pub struct Suspend { pub hide_cancel: Option, #[serde(skip_serializing_if = "false_or_empty")] pub continue_on_disapprove_timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skin: Option, +} + +/// How an approval request is presented, on the approval page and in Slack/Teams messages. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum ApprovalSkin { + Minimal, + /// A skin this server does not know renders as the detailed one rather than failing to + /// deserialize the whole flow, so a flow authored against a newer version still runs. + #[default] + #[serde(other)] + Detailed, } fn false_or_empty(v: &Option) -> bool { @@ -1081,7 +1100,8 @@ pub enum FlowModuleValue { omit_output_from_conversation: bool, /// When set, the agent brain config (provider/model/system prompt/etc.) and tools are /// resolved at runtime from this `ai_agent` resource path (hybrid linking). The module's - /// `input_transforms` then only carry the flow-local inputs (user_message/user_attachments). + /// `input_transforms` then only carry the flow-local inputs: user_message, + /// user_attachments, enabled_tools and the history inputs memory_id and previous_messages. #[serde(default, skip_serializing_if = "Option::is_none")] agent: Option, /// Binds an agent's tools to *this* flow's context, keyed by tool id then input key, without @@ -1365,6 +1385,23 @@ mod tests { assert_eq!(val.modules.len(), 1); } + #[test] + fn suspend_skin_unknown_value_falls_back_to_detailed() { + let skin_of = |skin: &str| { + let val: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": {"type": "identity"}, + "suspend": {"required_events": 1, "skin": skin} + }] + })) + .unwrap(); + val.modules[0].suspend.as_ref().unwrap().skin + }; + assert_eq!(skin_of("minimal"), Some(ApprovalSkin::Minimal)); + assert_eq!(skin_of("not_a_skin_yet"), Some(ApprovalSkin::Detailed)); + } + #[test] fn agent_tool_keeps_description_through_locking() { // #10244: the dependency job rebuilds each tool from its locked FlowModule; the diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index db20bc1251..5190efe380 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -632,6 +632,28 @@ pub enum JobPayload { }, } +impl JobPayload { + /// Whether the payload itself declares a dedicated worker, in which case `push` replaces + /// whatever tag it is handed and the caller's tag never reaches the queue. + /// + /// This reads what the payload carries, not what `push` will conclude: a `SingleStepFlow` + /// loads the flag from the script row at push time and reports `false` here. That only + /// matters to a caller reasoning about the tag, and for those the answer is the same either + /// way, since `push` replaces the tag in exactly the case this misses. + pub fn is_dedicated_worker(&self) -> bool { + let dedicated_worker = match self { + JobPayload::ScriptHash { dedicated_worker, .. } + | JobPayload::FlowScript { dedicated_worker, .. } + | JobPayload::Dependencies { dedicated_worker, .. } + | JobPayload::FlowDependencies { dedicated_worker, .. } + | JobPayload::Flow { dedicated_worker, .. } => dedicated_worker, + JobPayload::Code(raw) => &raw.dedicated_worker, + _ => &None, + }; + dedicated_worker.is_some_and(|x| x) + } +} + #[derive(Clone, Serialize, Deserialize, Debug)] pub struct SkipHandler { pub path: String, diff --git a/backend/windmill-types/src/more_serde.rs b/backend/windmill-types/src/more_serde.rs index d4b648d2d5..f9cbe739c5 100644 --- a/backend/windmill-types/src/more_serde.rs +++ b/backend/windmill-types/src/more_serde.rs @@ -38,6 +38,25 @@ pub fn is_default(t: &T) -> bool { &T::default() == t } +pub fn maybe_number<'de, T, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: FromStr + serde::Deserialize<'de>, + ::Err: Display, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumericOrString { + String(String), + RawT(T), + } + + match NumericOrString::::deserialize(deserializer)? { + NumericOrString::String(s) => T::from_str(&s).map_err(serde::de::Error::custom), + NumericOrString::RawT(i) => Ok(i), + } +} + pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -66,3 +85,60 @@ where NumericOrNull::Null => Ok(None), } } + +/// Deserializer for a doubly-optional field, so a struct can tell an absent key +/// (`None`) from an explicit `null` (`Some(None)`). +/// +/// Plain serde collapses both into the outer `None`, which makes the distinction +/// unusable exactly where it matters: a payload that omits a field means "leave it +/// alone", while one that sends `null` means "clear it". +/// +/// ```ignore +/// #[serde(default, deserialize_with = "double_option", skip_serializing_if = "Option::is_none")] +/// pub field: Option>, +/// ``` +pub fn double_option<'de, T, D>(deserializer: D) -> Result>, D::Error> +where + T: serde::Deserialize<'de>, + D: serde::Deserializer<'de>, +{ + serde::Deserialize::deserialize(deserializer).map(Some) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + #[derive(Deserialize)] + struct WithMaybeNumber { + #[serde(deserialize_with = "super::maybe_number")] + n: i64, + } + + #[test] + fn maybe_number_accepts_number() { + let v: WithMaybeNumber = serde_json::from_value(serde_json::json!({ "n": 12345 })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_accepts_string() { + let v: WithMaybeNumber = + serde_json::from_value(serde_json::json!({ "n": "12345" })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_rejects_non_numeric_string() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": "abc" })).is_err() + ); + } + + #[test] + fn maybe_number_rejects_null() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": null })).is_err() + ); + } +} diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 414650e846..9e29731505 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -16,7 +16,7 @@ use crate::{ runnable_settings::{ConcurrencySettings, DebouncingSettings}, }; -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)] pub struct ScriptModule { pub content: String, pub language: ScriptLang, @@ -507,6 +507,10 @@ pub struct ScriptHistory { pub deployment_msg: Option, #[serde(skip_serializing_if = "Option::is_none")] pub created_at: Option>, + /// Who deployed this version — the version picker names it, so a reader can + /// tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, } #[derive(Deserialize)] @@ -612,6 +616,7 @@ impl Hash for NewScript { self.priority.hash(state); self.timeout.hash(state); self.delete_after_use.hash(state); + self.delete_after_secs.hash(state); self.restart_unless_cancelled.hash(state); self.deployment_message.hash(state); self.visible_to_runner_only.hash(state); diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs index a9003a3712..e2e4ab866b 100644 --- a/backend/windmill-worker-volumes/src/lib.rs +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -179,11 +179,24 @@ pub fn interpolate_volume_name( result } +/// PHP's opening tag, which is case-insensitive and may be followed by code. +fn is_php_open_tag(trimmed_line: &str) -> bool { + trimmed_line + .get(..5) + .is_some_and(|p| p.eq_ignore_ascii_case(" volume: ` lines, +/// stopping at the first line that is neither blank nor a comment. A PHP script +/// opens with ` Vec { let mut volumes = Vec::new(); for line in content.lines() { let trimmed = line.trim(); - if trimmed.is_empty() { + if trimmed.is_empty() || is_php_open_tag(trimmed) { continue; } if !trimmed.starts_with(comment_prefix) { @@ -230,6 +243,17 @@ mod tests { ); } + #[test] + fn parse_php_volume_after_open_tag() { + let content = + " { - const url = readFileSync(args.path, "utf8"); - const req = await fetch(url, { - method: "GET", - headers: { - Authorization: "Bearer " + token, - }, - }); + // A stalled fetch would otherwise hold the whole build for bun's own 5-minute + // default, with nothing naming the script it was waiting on. + const RELATIVE_IMPORT_FETCH_TIMEOUT_MS = 120000; + + function relativeImportFetchError(url, e) { + const reason = + e?.name === "TimeoutError" + ? `no response within ${RELATIVE_IMPORT_FETCH_TIMEOUT_MS / 1000}s` + : String(e?.message ?? e); + return new Error(`Failed to fetch relative import at ${url}: ${reason}`); + } + + async function fetchRelativeImport(url) { + let req; + try { + req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + signal: AbortSignal.timeout(RELATIVE_IMPORT_FETCH_TIMEOUT_MS), + }); + } catch (e) { + throw relativeImportFetchError(url, e); + } if (!req.ok) { throw new Error( - `Failed to find relative import at ${url}`, - req.statusText + `Failed to find relative import at ${url} (status ${req.status} ${req.statusText})` ); } - const contents = await req.text(); + try { + return await req.text(); + } catch (e) { + throw relativeImportFetchError(url, e); + } + } + + build.onLoad({ filter: /.*\.url$/ }, async (args) => { + const url = readFileSync(args.path, "utf8"); + const contents = await fetchRelativeImport(url); return { contents: replaceRelativeImports(contents).contents, loader: "tsx", @@ -105,9 +130,11 @@ const p = { const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); const hash = TEMP_SCRIPT_REFS?.[normalized]; + // Lock generation substitutes `raw`: the dependency scan reads versions from the + // `pkg@version` specifiers in imported scripts, which `raw_unpinned` strips. const url = (isRelative - ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ? `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${file_path}/../${args.path}${endExt}` + : `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${args.path}${endExt}` ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js index a877266662..e9bb36c21e 100644 --- a/backend/windmill-worker/loader.bun.windows.js +++ b/backend/windmill-worker/loader.bun.windows.js @@ -101,6 +101,43 @@ const p = { return replaceRelativeImports(code); }); + // A stalled fetch would otherwise hold the whole build for bun's own 5-minute + // default, with nothing naming the script it was waiting on. + const RELATIVE_IMPORT_FETCH_TIMEOUT_MS = 120000; + + function relativeImportFetchError(url, e) { + const reason = + e?.name === "TimeoutError" + ? `no response within ${RELATIVE_IMPORT_FETCH_TIMEOUT_MS / 1000}s` + : String(e?.message ?? e); + return new Error(`Failed to fetch relative import at ${url}: ${reason}`); + } + + async function fetchRelativeImport(url) { + let req; + try { + req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + signal: AbortSignal.timeout(RELATIVE_IMPORT_FETCH_TIMEOUT_MS), + }); + } catch (e) { + throw relativeImportFetchError(url, e); + } + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url} (status ${req.status} ${req.statusText})` + ); + } + try { + return await req.text(); + } catch (e) { + throw relativeImportFetchError(url, e); + } + } + // Load windmill scripts by fetching from the API build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { // Extract temp_script_hash if embedded in the path by resolveWindmillImport @@ -110,18 +147,7 @@ const p = { : undefined; const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${scriptPath}` + (hashParam ? `?temp_script_hash=${hashParam}` : ""); - const req = await fetch(url, { - method: "GET", - headers: { - Authorization: "Bearer " + token, - }, - }); - if (!req.ok) { - throw new Error( - `Failed to find relative import at ${url} (status ${req.status})` - ); - } - const contents = await req.text(); + const contents = await fetchRelativeImport(url); return { contents: replaceRelativeImports(contents).contents, loader: "tsx", diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index ab10c36b86..bf67266d87 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -69,7 +69,11 @@ class WindmillFinder(MetaPathFinder): r = response.read().decode("utf-8") if r == "WINDMILL_IS_FOLDER": return ModuleSpec(name, WindmillLoader(name)) - with open(fullpath, "w+") as f: + # Python parses .py as UTF-8 regardless of locale, so the + # file has to be written as UTF-8. Without this the ANSI + # code page on a Windows worker re-encodes every + # non-ASCII literal and the import dies on a SyntaxError. + with open(fullpath, "w+", encoding="utf-8") as f: f.write(r) return spec_from_file_location(name, fullpath) except urllib.error.HTTPError as e: diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 8dccf691fa..e0971c08ce 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -64,16 +64,25 @@ pub async fn get_ducklake_from_agent_http( .await } +/// An agent worker authenticates as infrastructure, not as the job's user, so the job id travels +/// with the request: the server reads the job's owner from it and evaluates the data table's +/// tenants against them. A worker predating this sends neither, and the server fails it closed on +/// a data table under roles. #[allow(dead_code)] pub async fn get_datatable_resource_from_agent_http( client: &HttpClient, name: &str, w_id: &str, + role: Option<&str>, + job_id: &uuid::Uuid, ) -> anyhow::Result { + let role_query = role + .map(|r| format!("&role={}", urlencoding::encode(r))) + .unwrap_or_default(); client .get(&format!( - "/api/w/{}/agent_workers/get_datatable_resource/{}", - w_id, &name + "/api/w/{}/agent_workers/get_datatable_resource/{}?job_id={}{}", + w_id, &name, job_id, role_query )) .await } diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 9c15229ba6..7e775f34fe 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -33,7 +33,7 @@ use windmill_common::{ client::AuthedClient, db::DB, error::Error, - flow_conversations::MessageType, + flow_conversations::{MessageExtras, MessageType}, flow_status::AgentAction, flows::FlowModuleValue, worker::{to_raw_value, Connection}, @@ -74,6 +74,9 @@ pub struct ToolExecutionContext<'a> { pub stream_event_processor: Option<&'a StreamEventProcessor>, pub flow_context: &'a mut FlowContext, pub omit_output_from_conversation: bool, + /// The thinking that led to this round's calls, stored on the first tool row written. + /// None when the round wrote text, whose row carries it. + pub reasoning: Option, pub previous_result: &'a Option>, pub id_context: &'a Option, @@ -235,9 +238,24 @@ async fn execute_mcp_tool_call( update_flow_status_module_with_actions_success(ctx.db, parent_job, true).await?; } - // Add tool message to conversation if chat_input_enabled + // An MCP tool runs inside the agent's job, whose result holds every call of the + // turn and nothing tying one of them to this row: same job id for all of them, + // no call id on the row. Kept here so the card shows this call — and the row + // names that job, so retention sweeps it with every other row of the turn. let content = format!("Used {} tool", tool_call.function.name); - add_tool_message_to_chat(ctx, None, &content, true).await; + let agent_job_id = ctx.job.id; + add_tool_message_to_chat( + ctx, + Some(agent_job_id), + &content, + true, + Some(MessageExtras { + tool_arguments: Some(tool_call.function.arguments.clone()), + tool_result: Some(result_str), + ..Default::default() + }), + ) + .await; } Err(e) => { let error_msg = format!("MCP tool error: {}", e); @@ -271,8 +289,23 @@ async fn execute_mcp_tool_call( update_flow_status_module_with_actions_success(ctx.db, parent_job, false).await?; } - // Add tool message to conversation if chat_input_enabled - add_tool_message_to_chat(ctx, None, &error_msg, false).await; + // Add tool message to conversation if chat_input_enabled. The row is worded from + // the tool, like every other tool row, and the error it failed with is its result + // — the one field a call that produced nothing else still has something to put in. + let agent_job_id = ctx.job.id; + let content = format!("Error executing {}", tool_name); + add_tool_message_to_chat( + ctx, + Some(agent_job_id), + &content, + false, + Some(MessageExtras { + tool_arguments: Some(tool_call.function.arguments.clone()), + tool_result: Some(error_msg.clone()), + ..Default::default() + }), + ) + .await; } } @@ -680,8 +713,8 @@ async fn handle_tool_execution_error( update_flow_status_module_with_actions_success(ctx.db, parent_job, false).await?; } - // Add tool message to conversation if chat_input_enabled (error case) - add_tool_message_to_chat(ctx, Some(job_id), &error_message, false).await; + let (content, extras) = windmill_tool_row(tool_call, false, &error_message); + add_tool_message_to_chat(ctx, Some(job_id), &content, false, Some(extras)).await; Ok(()) } @@ -782,13 +815,17 @@ async fn handle_tool_execution_success( ..Default::default() }); - // Stream tool result (success case) + let (content, extras) = windmill_tool_row(tool_call, success, &tool_result); + + // The job ran; whether it ran successfully is `success`, and the row stored below is + // worded from it. The stream has to carry the same value, or the card the reader watches + // and the row that replaces it describe the same call differently. if let Some(stream_event_processor) = ctx.stream_event_processor { let tool_result_event = StreamingEvent::ToolResult { call_id: tool_call.id.clone(), function_name: tool_call.function.name.clone(), result: tool_result, - success: true, + success, }; stream_event_processor .send(tool_result_event, final_events_str) @@ -799,28 +836,56 @@ async fn handle_tool_execution_success( update_flow_status_module_with_actions_success(ctx.db, parent_job, success).await?; } - // Add tool message to conversation if chat_input_enabled + add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await; + + Ok(()) +} + +/// A Windmill tool's conversation row: worded from the tool, carrying the model's call and +/// the exact text the model got back, the same text agent memory keeps for that tool +/// message, so a card needs no job fetch. The call is the model's arguments, not the job's +/// args: the step's input transforms add inputs the model never wrote. +fn windmill_tool_row( + tool_call: &OpenAIToolCall, + success: bool, + sent_to_model: &str, +) -> (String, MessageExtras) { let content = if success { format!("Used {} tool", tool_call.function.name) } else { format!("Error executing {}", tool_call.function.name) }; - - add_tool_message_to_chat(ctx, Some(job_id), &content, success).await; - - Ok(()) + let extras = MessageExtras { + tool_arguments: Some(tool_call.function.arguments.clone()), + tool_result: Some(sent_to_model.to_string()), + ..Default::default() + }; + (content, extras) } /// Add tool message to conversation if chat is enabled async fn add_tool_message_to_chat( ctx: &mut ToolExecutionContext<'_>, + // The job this row belongs to: the tool's own where it has one, else the agent's, which + // is the job it ran inside. Every row names one so that retention collects the whole + // turn — `delete_jobs` removes messages by `job_id = ANY(..)` (there is no FK on the + // column; `drop_v2_job_side_table_cascades` dropped it), and a row naming no job would + // survive every purge and leave a conversation that can never become empty. tool_job_id: Option, content: &str, success: bool, + // The model's call and what it got back; every tool row carries both. + extras: Option, ) { if ctx.omit_output_from_conversation { return; } + let extras = match ctx.reasoning.take() { + Some(reasoning) => { + Some(MessageExtras { reasoning: Some(reasoning), ..extras.unwrap_or_default() }) + } + None => extras, + }; let chat_enabled = ctx .flow_context @@ -835,41 +900,74 @@ async fn add_tool_message_to_chat( .as_ref() .and_then(|fs| fs.memory_id) { - let db_clone = ctx.db.clone(); let effective_step_id = ctx .flow_step_id_override .or(ctx.job.flow_step_id.as_deref()); let step_name = get_step_name_from_flow(ctx.summary.as_deref(), effective_step_id); - let content = content.to_string(); - // Spawn task because we do not need to wait for the result - tokio::spawn(async move { - if let Err(e) = add_message_to_conversation( - &db_clone, - &memory_id, - tool_job_id, - &content, - MessageType::Tool, - &step_name, - success, - ) - .await - { - tracing::warn!( - "Failed to add tool message to conversation {}: {}", - memory_id, - e - ); - } - }); + // Awaited, not spawned: `created_seq` is the transcript's order, so a round's rows + // must commit in the order of its calls. Calls run one after another; running them + // in parallel would need their rows written in call order all the same. + if let Err(e) = add_message_to_conversation( + ctx.db, + &memory_id, + tool_job_id, + content, + MessageType::Tool, + &step_name, + success, + extras.as_ref(), + ) + .await + { + tracing::warn!( + "Failed to add tool message to conversation {}: {}", + memory_id, + e + ); + } } } } #[cfg(test)] mod tests { - use super::extract_ai_agent_output; + use super::{extract_ai_agent_output, windmill_tool_row}; use serde_json::value::RawValue; + use windmill_ai::ai_types::{OpenAIFunction, OpenAIToolCall}; + + #[test] + fn a_windmill_tool_row_carries_the_models_call_and_what_it_got_back() { + let tool_call = OpenAIToolCall { + id: "call_1".to_string(), + function: OpenAIFunction { + name: "get_price".to_string(), + arguments: r#"{"item":"widget"}"#.to_string(), + }, + r#type: "function".to_string(), + extra_content: None, + }; + + let (content, extras) = windmill_tool_row(&tool_call, true, r#"{"price":42}"#); + assert_eq!(content, "Used get_price tool"); + assert_eq!( + extras.tool_arguments.as_deref(), + Some(r#"{"item":"widget"}"#) + ); + assert_eq!(extras.tool_result.as_deref(), Some(r#"{"price":42}"#)); + + let (content, extras) = + windmill_tool_row(&tool_call, false, "Error running tool: ExecutionErr: boom"); + assert_eq!(content, "Error executing get_price"); + assert_eq!( + extras.tool_arguments.as_deref(), + Some(r#"{"item":"widget"}"#) + ); + assert_eq!( + extras.tool_result.as_deref(), + Some("Error running tool: ExecutionErr: boom") + ); + } #[test] fn extracts_only_the_output_of_an_agent_result() { diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 6e80551650..3efdc67a3d 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -13,7 +13,7 @@ use windmill_common::flows::FlowModuleValue; use windmill_common::{ db::DB, error::Error, - flow_conversations::{add_message_to_conversation_tx, MessageType}, + flow_conversations::{add_message_to_conversation_tx, MessageExtras, MessageType}, flow_status::AgentAction, flows::{InputTransform, Step}, jobs::JobKind, @@ -26,11 +26,11 @@ use windmill_queue::{flow_status::get_step_of_flow_status, MiniPulledJob}; use crate::parse_sig_of_lang; -pub fn parse_raw_script_schema( +pub async fn parse_raw_script_schema( content: &str, language: &ScriptLang, ) -> Result, Error> { - let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)? + let main_arg_signature = parse_sig_of_lang(content, Some(&language), None).await? .ok_or_else(|| Error::BadConfig(format!( "Cannot parse signature for language {:?}. The language parser may not be enabled in this build.", language @@ -154,6 +154,8 @@ pub async fn get_flow_job_runnable_and_raw_flow( pub struct FlowContext { pub flow_inputs: Option>>, pub flow_status: Option, + /// Path of the flow the run started from, which scopes a string memory id. + pub flow_path: Option, } /// Get flow context (chat settings + args + flow_status) from root flow's job data @@ -171,7 +173,8 @@ pub async fn get_flow_context(db: &DB, job: &MiniPulledJob) -> FlowContext { r#" SELECT j.args as "args: Json>>", - js.flow_status as "flow_status: Json" + js.flow_status as "flow_status: Json", + j.runnable_path FROM v2_job_status js INNER JOIN v2_job j ON j.id = js.id WHERE js.id = $1 @@ -184,6 +187,7 @@ pub async fn get_flow_context(db: &DB, job: &MiniPulledJob) -> FlowContext { Ok(Some(row)) => FlowContext { flow_inputs: row.args.map(|j| j.0), flow_status: row.flow_status.map(|j| j.0), + flow_path: row.runnable_path, }, Ok(None) => { tracing::warn!( @@ -209,6 +213,7 @@ pub async fn add_message_to_conversation( message_type: MessageType, step_name: &Option, success: bool, + extras: Option<&MessageExtras>, ) -> Result<(), Error> { let mut tx = db.begin().await?; add_message_to_conversation_tx( @@ -219,6 +224,7 @@ pub async fn add_message_to_conversation( message_type, step_name.as_deref(), success, + extras, ) .await?; tx.commit().await?; diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index d948fee088..3c3a26875b 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -26,6 +26,10 @@ use windmill_ai::{ image_handler::upload_image_to_s3, providers::{ create_chat_completions_query_builder, create_query_builder, is_chat_completions_only, + openai::{ + is_reasoning_summary_unavailable, rejects_reasoning_summary, + remember_reasoning_summary_unavailable, + }, remember_chat_completions_only, }, proxy::{ @@ -40,7 +44,7 @@ use windmill_common::{ client::AuthedClient, db::DB, error::{self, Error}, - flow_conversations::MessageType, + flow_conversations::{memory_key, MessageExtras, MessageType}, flow_status::AgentAction, flows::{AgentTool, FlowModule, FlowModuleValue, InputTransform, ToolValue}, get_latest_hash_for_path, @@ -49,7 +53,7 @@ use windmill_common::{ utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, Connection}, }; -use windmill_queue::{cancel_single_job, CanceledBy, MiniPulledJob}; +use windmill_queue::{append_logs, cancel_single_job, CanceledBy, MiniPulledJob}; use crate::{ ai::stream_event_processor::StreamEventProcessor, @@ -75,6 +79,15 @@ lazy_static::lazy_static! { const DEFAULT_MAX_AGENT_ITERATIONS: usize = 10; const HARD_MAX_AGENT_ITERATIONS: usize = 1000; +/// What a run stopped by `max_iterations` reports back. `Message` rather than +/// `OpenAIMessage` is load-bearing: `agent_action` is `skip_serializing` on the +/// latter and reaches JSON only through this wrapper, so serializing these raw +/// drops every tool name and job id and leaves the partial run unreadable. +#[derive(serde::Serialize)] +struct MaxIterPartialResult<'a> { + messages: Vec>, +} + fn strip_system_messages(messages: &[OpenAIMessage]) -> Vec { messages .iter() @@ -107,6 +120,148 @@ fn prepare_auto_memory_messages_for_persistence( non_system_messages[start_idx..].to_vec() } +/// The inputs a linked step supplies for itself; the resource holds the rest of the brain. +const FLOW_LOCAL_AGENT_KEYS: [&str; 5] = [ + "user_message", + "user_attachments", + "enabled_tools", + "memory_id", + "previous_messages", +]; + +/// The flow-local inputs that name a conversation, which a saved agent never carries. +const STEP_HISTORY_KEYS: [&str; 2] = ["memory_id", "previous_messages"]; + +/// Where one agent invocation's history comes from. +#[derive(Debug)] +enum HistorySource<'a> { + /// Supplied by the flow and replayed as is: memory is neither read nor written. + Messages(&'a [OpenAIMessage]), + Window { + memory_id: Uuid, + context_length: usize, + }, + Stateless, +} + +/// A step's memory id counts only as the step authored it. A static empty value is a form +/// placeholder, so it reads as unset rather than as an expression that evaluated to nothing, which +/// runs without memory; an AI-filled value would let the model choose which memory the agent reads. +fn keep_authored_memory_id( + args: &mut AIAgentArgs, + step_input_transforms: &HashMap, +) { + match step_input_transforms.get("memory_id") { + Some(InputTransform::Javascript { .. }) => {} + Some(InputTransform::Static { .. }) if args.memory_id.as_deref() != Some("") => {} + _ => args.memory_id = None, + } +} + +/// Reconciles the step's history inputs, the agent's memory policy and the run's memory id. A step +/// holds one of two shapes: an older `auto` or `manual` memory, read as the editor that wrote it +/// meant it, or the current setting plus the step's own history inputs. Also returns lines for the +/// job log: an input that went unused, or a policy that remembers ending up stateless. +fn resolve_history_source<'a>( + args: &'a AIAgentArgs, + run_memory_id: Option, + workspace_id: &str, + flow_path: &str, +) -> (HistorySource<'a>, Vec<&'static str>) { + let mut notes = Vec::new(); + let no_memory_id = "No memory id was passed to this run, so the agent runs without memory."; + match &args.memory { + // The step's own history inputs came after these, so a step that still holds one reads it + // alone: what it did before the editor offered them is what it keeps doing. + Some(Memory::Manual { messages }) => { + note_unread_step_inputs(&mut notes, args); + (HistorySource::Messages(messages), notes) + } + Some(Memory::Auto { context_length, memory_id }) => { + note_unread_step_inputs(&mut notes, args); + // An id baked in at save time only ever applied when the run carried none. + match run_memory_id.or(*memory_id) { + Some(memory_id) => ( + HistorySource::Window { memory_id, context_length: *context_length }, + notes, + ), + None => { + notes.push(no_memory_id); + (HistorySource::Stateless, notes) + } + } + } + Some(Memory::Window { context_length }) => { + if args + .previous_messages + .as_ref() + .is_some_and(|messages| !messages.is_empty()) + { + notes.push("Managed memory is on, so this step's previous messages are ignored."); + } + let memory_id = match args.memory_id.as_deref() { + Some("") => { + notes.push( + "This step's memory id evaluated to an empty value, so the agent runs without memory.", + ); + return (HistorySource::Stateless, notes); + } + Some(step_memory_id) => memory_key(workspace_id, flow_path, step_memory_id), + None => match run_memory_id { + Some(memory_id) => memory_id, + None => { + notes.push(no_memory_id); + return (HistorySource::Stateless, notes); + } + }, + }; + ( + HistorySource::Window { memory_id, context_length: *context_length }, + notes, + ) + } + Some(Memory::Off) | None => { + if args.memory_id.as_deref().is_some_and(|id| !id.is_empty()) { + notes.push("Managed memory is off, so this step's memory id is ignored."); + } + match &args.previous_messages { + Some(messages) => (HistorySource::Messages(messages), notes), + None => (HistorySource::Stateless, notes), + } + } + } +} + +/// An older memory setting reads neither history input, which is only visible in the job log: the +/// editor offers them on a step that has been moved to the current settings. +fn note_unread_step_inputs(notes: &mut Vec<&'static str>, args: &AIAgentArgs) { + if args.memory_id.as_deref().is_some_and(|id| !id.is_empty()) { + notes.push("This step uses an older memory setting, so its memory id is not read."); + } + if args + .previous_messages + .as_ref() + .is_some_and(|messages| !messages.is_empty()) + { + notes + .push("This step uses an older memory setting, so its previous messages are not read."); + } +} + +/// Whether a request has something to ask the model. Only text output sends previous messages, so +/// an image prompt comes from the user message alone. An empty list is no conversation, except +/// under a legacy `manual` memory, which ran on whatever list it held. +fn has_prompt( + history: &HistorySource, + has_user_message: bool, + is_text_output: bool, + legacy_list: bool, +) -> bool { + has_user_message + || (is_text_output + && (legacy_list || matches!(history, HistorySource::Messages(m) if !m.is_empty()))) +} + fn find_module_by_id( modules: &Vec, target_id: &str, @@ -132,14 +287,16 @@ async fn find_ai_agent_tool_module_in_parent_agent( return Ok(None); }; - let FlowModuleValue::AIAgent { tools, agent, .. } = parent_agent_module.get_value()? else { + let FlowModuleValue::AIAgent { tools, agent, tool_inputs, .. } = + parent_agent_module.get_value()? + else { return Ok(None); }; // A linked parent carries no tools on the module (they live in the resource, resolved only in // the main execution branch). Resolve them from the resource here too, so a nested agent tool // of a saved+linked agent can still be located when it runs as its own job. - let tools = if let Some(agent_ref) = agent.as_deref() { + let mut tools = if let Some(agent_ref) = agent.as_deref() { let agent_path = agent_ref .trim_start_matches("$res:") .trim_start_matches("res://"); @@ -166,6 +323,9 @@ async fn find_ai_agent_tool_module_in_parent_agent( } else { tools }; + // The nested job reads its history inputs from the tool's transforms, which must carry the + // host flow's bindings as the parent evaluated them. + overlay_tool_inputs(&mut tools, &tool_inputs); for tool in tools { if tool.id == tool_module_id { @@ -243,6 +403,88 @@ fn overlay_tool_inputs( } } +/// What every websearch entry is named by, whatever label it carries. Web search reaches the model +/// as a provider capability rather than a tool, so it has no model-facing name of its own, and the +/// editor's label is not one: a flow module tool could carry the same one, and enabling that tool +/// would then silently turn web search on with it. +/// +/// Reserved, on the `__wm_` prefix this codebase uses for names it keeps for itself, and held that +/// way by `flow_module_tool_name` refusing to advertise a tool that takes it. +const WEBSEARCH_ENABLED_NAME: &str = "__wm_web_search"; + +/// The name a flow module tool is advertised to the model under. +/// +/// Rejected rather than skipped: a tool the model is never shown is a tool the agent silently does +/// not have, and a run that quietly drops one is harder to explain than a run that will not start. +fn flow_module_tool_name(summary: Option<&str>) -> Result<&str, Error> { + match summary { + Some(name) if name == WEBSEARCH_ENABLED_NAME => Err(Error::internal_err(format!( + "Invalid tool name: {name:?} is reserved for enabling web search" + ))), + Some(name) if TOOL_NAME_REGEX.is_match(name) => Ok(name), + other => Err(Error::internal_err(format!("Invalid tool name: {other:?}"))), + } +} + +/// The name a run enables a roster entry by: the name the model is shown, except for an entry the +/// model is shown nothing of, which cannot be named by a label others may share. An MCP server is +/// named by the resource it points at, web search by `WEBSEARCH_ENABLED_NAME`. +/// +/// The MCP path is bare. The roster stores it as authored, `$res:` and all, but a name is an +/// argument value and one carrying that prefix is resolved to the resource itself before the worker +/// is handed its args, so the prefixed form is not something the list can hold. +fn tool_enabled_name(tool: &AgentTool) -> Option<&str> { + match &tool.value { + ToolValue::Mcp(mcp) => Some(mcp.resource_path.trim_start_matches("$res:")), + ToolValue::Websearch(_) => Some(WEBSEARCH_ENABLED_NAME), + _ => tool.summary.as_deref(), + } +} + +/// The roster a run advertises, given the names it enabled. +/// +/// `None` advertises the whole roster, which is what every agent written before the field existed +/// relies on; an empty list advertises nothing. +/// +/// Whole entries, decided before any of them is resolved: an MCP server a run switched off is never +/// contacted, and reading its resource, refreshing its token and opening a client are each things +/// that can fail a run. Which tools a server exposes stays its entry's own `include_tools` / +/// `exclude_tools`, the only place that choice is made. +fn narrow_roster(tools: Vec, enabled_tools: Option<&[String]>) -> Vec { + let Some(enabled) = enabled_tools else { + return tools; + }; + tools + .into_iter() + .filter(|t| tool_enabled_name(t).is_some_and(|name| enabled.iter().any(|n| n == name))) + .collect() +} + +/// The log line for names in `enabled_tools` that name nothing on the agent, if any. The list can +/// be computed per run, so a name that has since been renamed away must not fail the step — but it +/// would otherwise silently narrow the agent, so the run says how many of its names matched nothing. +/// +/// Counted, never quoted: a name is an argument value, and one written as `$var:path` reaches the +/// worker already replaced by the variable's own value. Quoting it would write that value to the +/// job log, where the masks a job registers never reach it — they are applied to a subprocess's +/// output in `handle_child` and nowhere else, and `append_logs` stores what it is given verbatim. +fn unmatched_enabled_tools_message( + enabled_tools: &[String], + advertised: &[&str], +) -> Option { + let unmatched = enabled_tools + .iter() + .filter(|name| !advertised.contains(&name.as_str())) + .count(); + if unmatched == 0 { + return None; + } + let subject = if unmatched == 1 { "name" } else { "names" }; + Some(format!( + "--- ENABLED TOOLS: {unmatched} {subject} named no tool of this agent and had no effect ---\n" + )) +} + pub async fn handle_ai_agent_job( // connection conn: &Connection, @@ -370,6 +612,7 @@ pub async fn handle_ai_agent_job( omit_output_from_conversation, agent, tool_inputs, + input_transforms: step_input_transforms, .. } = module.get_value()? else { @@ -378,10 +621,13 @@ pub async fn handle_ai_agent_job( )); }; - // A linked step takes its brain and tools from the resource and keeps only the flow-local - // inputs (user_message/user_attachments) of its own; both stay rigid, so the one thing it may - // bind to this flow is the tools' inputs, overlaid from `tool_inputs` below. - let (args, tools): (AIAgentArgs, Vec) = if let Some(agent_ref) = agent.as_deref() { + // A linked step takes its brain and tools from the resource and keeps only its own flow-local + // inputs. The brain and the roster stay rigid; what the step binds to this flow is the message + // it asks, which of those tools this use may call, the conversation it is part of (its memory + // id and previous messages), and the tools' own inputs — the last overlaid from `tool_inputs` + // below. + let (mut args, tools): (AIAgentArgs, Vec) = if let Some(agent_ref) = agent.as_deref() + { let agent_path = agent_ref .trim_start_matches("$res:") .trim_start_matches("res://"); @@ -413,6 +659,12 @@ pub async fn handle_ai_agent_job( None => Vec::new(), }; overlay_tool_inputs(&mut tools, &tool_inputs); + // The resource is not validated against a schema, so a history input it happens to carry + // is dropped before interpolation, where a bad `$res:` in it would fail the step. The + // other flow-local keys stay: a resource's own user message is the step's fallback. + for key in STEP_HISTORY_KEYS { + config.remove(key); + } let brain = transform_json_value( "ai_agent", client, @@ -434,7 +686,7 @@ pub async fn handle_ai_agent_job( // Only after interpolating the resource: these are caller-controlled and already resolved by // build_args_map, so passing them through it again would expand contextual values — // `$WM_TOKEN` in a user message would reach the model provider. - for key in ["user_message", "user_attachments"] { + for key in FLOW_LOCAL_AGENT_KEYS { if let Some(v) = local_args.get(key) { brain.insert( key.to_string(), @@ -459,6 +711,8 @@ pub async fn handle_ai_agent_job( (args, tools) }; + keep_authored_memory_id(&mut args, &step_input_transforms); + // Nesting is capped at flow → agent → nested agent. When this job is itself a nested tool, // a linked resource's tool set may still contain AIAgent tools (the editor can't constrain a // shared resource); don't advertise them — invoking one would only fail the depth check as a @@ -477,6 +731,20 @@ pub async fn handle_ai_agent_job( tools }; + // Narrow the roster to the tools this run enabled, before the loop below pays a script or hub + // fetch per tool. + let enabled_tools = args.enabled_tools.as_deref(); + // Taken before the narrowing consumes the roster, and only by a run that narrows: they are what + // its names are matched against, so they are also what tells it a name matched nothing. + let roster_names: Vec = match enabled_tools { + Some(_) => tools + .iter() + .filter_map(|t| tool_enabled_name(t).map(str::to_string)) + .collect(), + None => Vec::new(), + }; + let tools = narrow_roster(tools, enabled_tools); + // Separate Windmill tools from MCP tools, websearch, and extract MCP resource configs let mut windmill_modules: Vec = Vec::new(); // Explicit per-tool descriptions keyed by tool id. When set, these override the @@ -541,12 +809,7 @@ pub async fn handle_ai_agent_job( let job = job; let user_description = tool_descriptions.get(&t.id).cloned(); async move { - let Some(summary) = t.summary.as_ref().filter(|s| TOOL_NAME_REGEX.is_match(s)) else { - return Err(Error::internal_err(format!( - "Invalid tool name: {:?}", - t.summary - ))); - }; + let summary = flow_module_tool_name(t.summary.as_deref())?; // Extract schema, input_transforms, and an auto-derived description from the module value let module_value = t.get_value()?; @@ -621,7 +884,7 @@ pub async fn handle_ai_agent_job( (schema, input_transforms, derived_description) } FlowModuleValue::RawScript { content, language, input_transforms, .. } => { - let schema = Some(parse_raw_script_schema(&content, &language)?); + let schema = Some(parse_raw_script_schema(&content, &language).await?); (schema, input_transforms, None) } FlowModuleValue::AIAgent { input_transforms, .. } => { @@ -657,7 +920,7 @@ pub async fn handle_ai_agent_job( def: ToolDef { r#type: "function".to_string(), function: ToolDefFunction { - name: summary.clone(), + name: summary.to_string(), description: Some(description), parameters: schema.unwrap_or_else(|| { to_raw_value(&serde_json::json!({ @@ -687,6 +950,22 @@ pub async fn handle_ai_agent_job( HashMap::new() }; + if let Some(enabled) = enabled_tools { + let matchable: Vec<&str> = roster_names.iter().map(|s| s.as_str()).collect(); + windmill_common::feature_usage::log_feature_usage( + "ai_agent", + "dynamic_tools", + if tools.is_empty() && !has_websearch { + "no_tools" + } else { + "tools" + }, + ); + if let Some(message) = unmatched_enabled_tools_message(enabled, &matchable) { + append_logs(&job.id, &job.workspace_id, message, conn).await; + } + } + let mut inner_occupancy_metrics = occupancy_metrics.clone(); let stream_notifier = StreamNotifier::new(conn, job); @@ -898,8 +1177,18 @@ pub async fn run_agent( // Fetch flow context for input transforms context, chat and memory let mut flow_context = get_flow_context(db, job).await; - // Determine if we're using manual messages (which bypasses memory) - let use_manual_messages = matches!(args.memory, Some(Memory::Manual { .. })); + // The run's memory id is also the chat conversation id, which a step's own memory id never + // replaces. + let conversation_id = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id); + let (history, history_notes) = resolve_history_source( + args, + conversation_id, + &job.workspace_id, + flow_context.flow_path.as_deref().unwrap_or_default(), + ); // Check if user_message is provided and non-empty let has_user_message = args @@ -908,63 +1197,63 @@ pub async fn run_agent( .map(|m| !m.is_empty()) .unwrap_or(false); - // Validate: at least one of memory with manual messages or user_message must be provided - if !use_manual_messages && !has_user_message { - return Err(Error::internal_err( - "Either 'memory' with manual messages or 'user_message' must be provided".to_string(), - )); - } - let is_text_output = output_type == &OutputType::Text; - // Flow-level memory_id (from chat mode) takes precedence over step-level memory_id - let memory_id = flow_context - .flow_status - .as_ref() - .and_then(|fs| fs.memory_id) - .or_else(|| { - // Extract memory_id from Memory::Auto if present - match &args.memory { - Some(Memory::Auto { memory_id, .. }) => *memory_id, - _ => None, - } - }); + if is_text_output { + for note in &history_notes { + append_logs(&job.id, &job.workspace_id, format!("{note}\n"), conn).await; + } + } else if !matches!(args.memory, None | Some(Memory::Off)) + || args.memory_id.is_some() + || args.previous_messages.is_some() + { + append_logs( + &job.id, + &job.workspace_id, + "Image output sends no history, so memory and previous messages are not read.\n", + conn, + ) + .await; + } + + // A `manual` memory sent whatever list it held, an empty one included, so a step that still has + // one keeps running without a user message. + let legacy_list = matches!(args.memory, Some(Memory::Manual { .. })); + if !has_prompt(&history, has_user_message, is_text_output, legacy_list) { + let missing = if !is_text_output { + "'user_message' must be provided for image output" + } else if matches!( + args.memory, + Some(Memory::Window { .. } | Memory::Auto { .. }) + ) { + "'user_message' must be provided while managed memory is on" + } else { + "Either 'previous_messages' or 'user_message' must be provided" + }; + return Err(Error::internal_err(missing.to_string())); + } - // Load messages based on history mode if matches!(output_type, OutputType::Text) { - match &args.memory { - Some(Memory::Manual { messages: manual_messages }) => { - // Use explicitly provided messages (bypass memory) - if !manual_messages.is_empty() { - messages.extend(manual_messages.clone()); - } - } - Some(Memory::Auto { context_length, .. }) => { - // Auto mode: load from memory + match &history { + HistorySource::Messages(provided) => messages.extend(provided.iter().cloned()), + HistorySource::Window { memory_id, context_length } => { if let Some(step_id) = effective_flow_step_id { - if let Some(memory_id) = memory_id { - // Read messages from memory - match read_from_memory(db, &job.workspace_id, memory_id, step_id).await { - Ok(Some(loaded_messages)) => { - let messages_to_load = prepare_auto_memory_messages_for_request( - &loaded_messages, - *context_length, - ); - messages.extend(messages_to_load); - } - Ok(None) => {} - Err(e) => { - tracing::error!( - "Failed to read memory for step {}: {}", - step_id, - e - ); - } + match read_from_memory(db, &job.workspace_id, *memory_id, step_id).await { + Ok(Some(loaded_messages)) => { + let messages_to_load = prepare_auto_memory_messages_for_request( + &loaded_messages, + *context_length, + ); + messages.extend(messages_to_load); + } + Ok(None) => {} + Err(e) => { + tracing::error!("Failed to read memory for step {}: {}", step_id, e); } } } } - _ => {} + HistorySource::Stateless => {} } } @@ -1099,10 +1388,11 @@ pub async fn run_agent( // For non-Anthropic providers, response_format is handled by the query builder } - let user_wants_streaming = args.streaming.unwrap_or(false); + let user_wants_streaming = streaming_requested(args.streaming); *has_stream = user_wants_streaming && is_text_output; let mut final_events_str = String::new(); + let mut final_reasoning = String::new(); // Always create a StreamEventProcessor for text output (use silent mode if user doesn't want streaming) let stream_event_processor = if is_text_output { @@ -1191,6 +1481,10 @@ pub async fn run_agent( attachments: args.user_attachments.as_deref(), has_websearch, prompt_cache_key: include_prompt_cache_key.then_some(prompt_cache_key.as_str()), + reasoning_summary: !is_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ), }; // A worker cannot run the client credentials exchange, so an OAuth resource @@ -1241,7 +1535,8 @@ pub async fn run_agent( // An endpoint can reject the request shape rather than the model: // `stream_options` and `prompt_cache_key`, which not every OpenAI-compatible - // gateway accepts, and the route itself, when an Azure resource is outside + // gateway accepts, a reasoning summary, which OpenAI refuses to unverified + // organizations, and the route itself, when an Azure resource is outside // the Responses API's model/region matrix. Each is retried once with that // part dropped. // Set where the route is found to be absent, and read once the fallback has @@ -1298,6 +1593,9 @@ pub async fn run_agent( && status.as_u16() == 400 && text.contains("prompt_cache_key"); + let summary_refused = build_args.reasoning_summary + && rejects_reasoning_summary(status.as_u16(), &text); + // Only the first call of the step may re-route: an endpoint that // does not serve this API rejects that one already, whereas a // rejection once the conversation is under way is about the @@ -1321,6 +1619,15 @@ pub async fn run_agent( ); include_prompt_cache_key = false; build_args.prompt_cache_key = None; + } else if summary_refused { + tracing::info!( + "Retrying request without the reasoning summary the endpoint refused" + ); + remember_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ); + build_args.reasoning_summary = false; } else if route_unserved { tracing::info!( "Endpoint rejected the request ({}), falling back to chat/completions", @@ -1355,6 +1662,7 @@ pub async fn run_agent( match parsed { ParsedResponse::Text { content: response_content, + reasoning: response_reasoning, tool_calls, events_str, annotations, @@ -1371,6 +1679,7 @@ pub async fn run_agent( if let Some(events_str) = events_str { final_events_str.push_str(&events_str); } + append_reasoning(&mut final_reasoning, response_reasoning.as_deref()); // Add websearch tool message if websearch was used if used_websearch { @@ -1389,30 +1698,35 @@ pub async fn run_agent( ..Default::default() }); if persist_output_to_conversation { - if let Some(memory_id) = memory_id { - let agent_job_id = job.id; - let db_clone = db.clone(); - let message_content = "Used websearch tool successfully".to_string(); - let step_name = step_name.clone(); - tokio::spawn(async move { - if let Err(e) = add_message_to_conversation( - &db_clone, - &memory_id, - Some(agent_job_id), - &message_content, - MessageType::Tool, - &step_name, - true, - ) - .await - { - tracing::warn!( - "Failed to add websearch tool message to conversation {}: {}", - memory_id, - e - ); - } + if let Some(conversation_id) = conversation_id { + // The search ran inside the provider's call, so this job's args + // describe the agent, not the search: its sources reach the row + // only if they are written here. + let extras = (!annotations.is_empty()).then(|| MessageExtras { + tool_result: serde_json::to_string(&annotations).ok(), + ..Default::default() }); + // Awaited like every row of the loop, so rows commit in turn order. + // Worded like every other tool row, so a reader recovers the tool + // name from the sentence. + if let Err(e) = add_message_to_conversation( + db, + &conversation_id, + Some(job.id), + "Used websearch tool", + MessageType::Tool, + &step_name, + true, + extras.as_ref(), + ) + .await + { + tracing::warn!( + "Failed to add websearch tool message to conversation {}: {}", + conversation_id, + e + ); + } } } } @@ -1441,32 +1755,30 @@ pub async fn run_agent( // Add assistant message to conversation if chat_input_enabled if persist_output_to_conversation && !response_content.is_empty() { - if let Some(memory_id) = memory_id { - let agent_job_id = job.id; - let db_clone = db.clone(); - let message_content = response_content.clone(); - let step_name = step_name.clone(); - - // Spawn task because we do not need to wait for the result - tokio::spawn(async move { - if let Err(e) = add_message_to_conversation( - &db_clone, - &memory_id, - Some(agent_job_id), - &message_content, - MessageType::Assistant, - &step_name, - true, - ) - .await - { - tracing::warn!( - "Failed to add assistant message to conversation {}: {}", - memory_id, - e - ); - } + if let Some(conversation_id) = conversation_id { + // This iteration's thinking goes on the answer's row; the job + // result only keeps the turn's thinking as one string. + let extras = response_reasoning.clone().map(|reasoning| { + MessageExtras { reasoning: Some(reasoning), ..Default::default() } }); + if let Err(e) = add_message_to_conversation( + db, + &conversation_id, + Some(job.id), + response_content, + MessageType::Assistant, + &step_name, + true, + extras.as_ref(), + ) + .await + { + tracing::warn!( + "Failed to add assistant message to conversation {}: {}", + conversation_id, + e + ); + } } } } @@ -1482,10 +1794,6 @@ pub async fn run_agent( step_id: Option<&'a str>, result: MaxIterPartialResult<'a>, } - #[derive(serde::Serialize)] - struct MaxIterPartialResult<'a> { - messages: &'a [OpenAIMessage], - } return Err(Error::ExecutionRawError( serde_json::value::to_raw_value(&MaxIterError { message: format!( @@ -1494,7 +1802,15 @@ pub async fn run_agent( ), name: "ExecutionErr", step_id: effective_flow_step_id, - result: MaxIterPartialResult { messages: &messages }, + result: MaxIterPartialResult { + messages: messages + .iter() + .map(|m| Message { + message: m, + agent_action: m.agent_action.as_ref(), + }) + .collect(), + }, })?, )); } @@ -1505,6 +1821,18 @@ pub async fn run_agent( ..Default::default() }); + // A round's thinking is stored on one row, the first the round writes, which is + // where the stream shows it: its text row when it wrote text, else the row of + // its first call — the answer row below when that call is the structured-output + // tool. Two rows carrying it would show it twice after a reload. + let call_reasoning = response_reasoning + .clone() + .filter(|_| response_content.as_deref().unwrap_or("").is_empty()); + let structured_output_first = structured_output_tool_name + .as_ref() + .zip(tool_calls.first()) + .map_or(false, |(name, tc)| tc.function.name == *name); + // Handle tool calls using extracted tools module let tool_execution_ctx = ToolExecutionContext { db, @@ -1523,6 +1851,11 @@ pub async fn run_agent( stream_event_processor: stream_event_processor.as_ref(), flow_context: &mut flow_context, omit_output_from_conversation, + reasoning: if structured_output_first { + None + } else { + call_reasoning.clone() + }, previous_result: &previous_result, id_context: &id_context, tool_abort_handles: tool_abort_handles.clone(), @@ -1541,6 +1874,41 @@ pub async fn run_agent( .await?; messages.extend(tool_messages); + + // A structured answer is the arguments of the structured-output tool call, + // on which the loop ends without a text iteration, so its row is written here. + if tool_used_structured_output && persist_output_to_conversation { + if let (Some(conversation_id), Some(OpenAIContent::Text(answer))) = + (conversation_id, tool_content.as_ref()) + { + let extras = call_reasoning + .clone() + .filter(|_| structured_output_first) + .map(|reasoning| MessageExtras { + reasoning: Some(reasoning), + ..Default::default() + }); + if let Err(e) = add_message_to_conversation( + db, + &conversation_id, + Some(job.id), + answer, + MessageType::Assistant, + &step_name, + true, + extras.as_ref(), + ) + .await + { + tracing::warn!( + "Failed to add structured answer to conversation {}: {}", + conversation_id, + e + ); + } + } + } + if let Some(tc) = tool_content { content = Some(tc); } @@ -1560,10 +1928,7 @@ pub async fn run_agent( // Add assistant message to conversation if chat_input_enabled if persist_output_to_conversation { - if let Some(memory_id) = memory_id { - let agent_job_id = job.id; - let db_clone = db.clone(); - + if let Some(conversation_id) = conversation_id { // Create extended version with type discriminator for conversation storage // This avoids conflicts with outputs that are of the same format as S3 objects let s3_with_type = S3ObjectWithType { @@ -1574,26 +1939,24 @@ pub async fn run_agent( let message_content = serde_json::to_string(&s3_with_type) .unwrap_or_else(|_| content.get().to_string()); - // Spawn task because we do not need to wait for the result - tokio::spawn(async move { - if let Err(e) = add_message_to_conversation( - &db_clone, - &memory_id, - Some(agent_job_id), - &message_content, - MessageType::Assistant, - &step_name, - true, - ) - .await - { - tracing::warn!( - "Failed to add assistant message to conversation {}: {}", - memory_id, - e - ); - } - }); + if let Err(e) = add_message_to_conversation( + db, + &conversation_id, + Some(job.id), + &message_content, + MessageType::Assistant, + &step_name, + true, + None, + ) + .await + { + tracing::warn!( + "Failed to add assistant message to conversation {}: {}", + conversation_id, + e + ); + } } } @@ -1646,13 +2009,10 @@ pub async fn run_agent( } } - // Persist complete conversation to memory at the end (only if in auto mode with context length) - // Skip memory persistence if using manual messages (bypass memory entirely) - // final_messages contains the complete history (old messages + new ones) - if matches!(output_type, OutputType::Text) && !use_manual_messages { - if let Some(Memory::Auto { context_length, .. }) = &args.memory { + // final_messages holds the complete history: what was loaded plus this run's messages + if matches!(output_type, OutputType::Text) { + if let HistorySource::Window { memory_id, context_length } = &history { if let Some(step_id) = effective_flow_step_id { - // Extract OpenAIMessages from final_messages let all_messages: Vec = final_messages.iter().map(|m| m.message.clone()).collect(); @@ -1662,23 +2022,21 @@ pub async fn run_agent( *context_length, ); - if let Some(memory_id) = memory_id { - if let Err(e) = write_to_memory( - db, - &job.workspace_id, - memory_id, + if let Err(e) = write_to_memory( + db, + &job.workspace_id, + *memory_id, + step_id, + &messages_to_persist, + ) + .await + { + tracing::error!( + "Failed to persist {} messages to memory for step {}: {}", + messages_to_persist.len(), step_id, - &messages_to_persist, - ) - .await - { - tracing::error!( - "Failed to persist {} messages to memory for step {}: {}", - messages_to_persist.len(), - step_id, - e - ); - } + e + ); } } } @@ -1693,6 +2051,7 @@ pub async fn run_agent( } else { None }, + reasoning: (!final_reasoning.is_empty()).then_some(final_reasoning), usage: if final_usage.as_ref().map(|u| u.is_empty()).unwrap_or(true) { None } else { @@ -1701,6 +2060,30 @@ pub async fn run_agent( })) } +/// Whether the step asked for its answer as it is generated. Absence means on, matching the +/// schema's own default: a step that never wrote the key never had an opinion, and an answer +/// arriving as it is written is what people expect. Only an explicit `false` holds it back. +/// +/// The chat surfaces decide whether to open a stream from their own reading of the same config, +/// and a surface that opens one for an answer sent in a single piece re-runs the flow when the +/// connection times out. So this default is half of a contract, not a local preference. +fn streaming_requested(streaming: Option) -> bool { + streaming.unwrap_or(true) +} + +/// Add one iteration's thinking to the step's. Every iteration thinks, and a tool-call +/// iteration's thinking is what led to the call, so the result keeps all of them in order, +/// blank-line separated, rather than only the answering turn's. +fn append_reasoning(accumulated: &mut String, reasoning: Option<&str>) { + let Some(reasoning) = reasoning.map(str::trim).filter(|r| !r.is_empty()) else { + return; + }; + if !accumulated.is_empty() { + accumulated.push_str("\n\n"); + } + accumulated.push_str(reasoning); +} + #[cfg(test)] mod tests { use super::*; @@ -1713,6 +2096,272 @@ mod tests { } } + #[derive(Debug, PartialEq)] + enum Resolved { + Messages(usize), + Window(Uuid, usize), + Stateless { noted: bool }, + } + + /// Every memory shape a worker may still read, resolved against a run with or without a + /// memory id. The hashed id is pinned: changing it detaches memories stored under string ids. + #[test] + fn history_source_resolves_every_memory_shape() { + use serde_json::json; + let run = Uuid::from_u128(1); + let baked = Uuid::from_u128(2); + let cust_1 = Uuid::parse_str("0168fcea-ffa7-5c15-bdb0-7709bb5f540d").unwrap(); + let window = json!({ "kind": "window", "context_length": 10 }); + let message = json!([{ "role": "user", "content": "earlier" }]); + let two_messages = json!([ + { "role": "user", "content": "earlier" }, + { "role": "assistant", "content": "reply" } + ]); + let cases = [ + ( + "absent memory is off", + json!({}), + Some(run), + Resolved::Stateless { noted: false }, + ), + ( + "legacy off", + json!({ "memory": { "kind": "off" } }), + Some(run), + Resolved::Stateless { noted: false }, + ), + ( + "legacy auto prefers the run's id", + json!({ "memory": { "kind": "auto", "context_length": 4, "memory_id": baked } }), + Some(run), + Resolved::Window(run, 4), + ), + ( + "legacy auto falls back to its baked id", + json!({ "memory": { "kind": "auto", "context_length": 4, "memory_id": baked } }), + None, + Resolved::Window(baked, 4), + ), + ( + "legacy auto with an empty baked id uses the run's", + json!({ "memory": { "kind": "auto", "context_length": 4, "memory_id": "" } }), + Some(run), + Resolved::Window(run, 4), + ), + ( + "legacy auto with an empty baked id and no run id is stateless", + json!({ "memory": { "kind": "auto", "context_length": 4, "memory_id": " " } }), + None, + Resolved::Stateless { noted: true }, + ), + ( + "legacy auto without a length is off", + json!({ "memory": { "kind": "auto", "memory_id": baked } }), + Some(run), + Resolved::Stateless { noted: false }, + ), + ( + "a cleared count is off", + json!({ "memory": { "kind": "window", "context_length": null } }), + Some(run), + Resolved::Stateless { noted: false }, + ), + ( + "legacy manual replays its messages", + json!({ "memory": { "kind": "manual", "messages": message } }), + Some(run), + Resolved::Messages(1), + ), + ( + "window keeps the run's memory", + json!({ "memory": window }), + Some(run), + Resolved::Window(run, 10), + ), + ( + "window without a memory id is stateless", + json!({ "memory": window }), + None, + Resolved::Stateless { noted: true }, + ), + ( + "a step memory id overrides the run's", + json!({ "memory": window, "memory_id": "cust_1" }), + Some(run), + Resolved::Window(cust_1, 10), + ), + ( + "a uuid step memory id is used as is", + json!({ "memory": window, "memory_id": baked.to_string() }), + Some(run), + Resolved::Window(baked, 10), + ), + ( + "a step memory id evaluating to null is stateless", + json!({ "memory": window, "memory_id": null }), + Some(run), + Resolved::Stateless { noted: true }, + ), + ( + "an off policy ignores the step memory id, and says so", + json!({ "memory": { "kind": "off" }, "memory_id": "cust_1" }), + Some(run), + Resolved::Stateless { noted: true }, + ), + ( + "managed memory ignores the step's previous messages", + json!({ "memory": window, "memory_id": "cust_1", "previous_messages": message }), + Some(run), + Resolved::Window(cust_1, 10), + ), + ( + "memory that is off sends the step's previous messages", + json!({ "previous_messages": message }), + Some(run), + Resolved::Messages(1), + ), + ( + "a previous messages expression that evaluated to null is no history", + json!({ "previous_messages": null }), + Some(run), + Resolved::Stateless { noted: false }, + ), + ( + "a legacy manual list ignores the step's previous messages", + json!({ "memory": { "kind": "manual", "messages": message }, "previous_messages": two_messages }), + Some(run), + Resolved::Messages(1), + ), + ( + "legacy auto ignores a step memory id", + json!({ "memory": { "kind": "auto", "context_length": 4, "memory_id": baked }, "memory_id": "cust_1" }), + None, + Resolved::Window(baked, 4), + ), + ]; + for (name, history, run_memory_id, expected) in cases { + let mut raw = json!({ "provider": { "kind": "openai", "resource": {}, "model": "m" } }); + raw.as_object_mut() + .unwrap() + .extend(history.as_object().unwrap().clone()); + let args: AIAgentArgs = serde_json::from_value(raw).unwrap(); + let resolved = match resolve_history_source(&args, run_memory_id, "ws", "f/flow") { + (HistorySource::Messages(m), _) => Resolved::Messages(m.len()), + (HistorySource::Window { memory_id, context_length }, _) => { + Resolved::Window(memory_id, context_length) + } + (HistorySource::Stateless, notes) => { + Resolved::Stateless { noted: !notes.is_empty() } + } + }; + assert_eq!(resolved, expected, "{name}"); + } + } + + /// A placeholder the form seeds must not read as a memory id that evaluated to nothing, which + /// would turn memory off for the step. + #[test] + fn only_an_expression_can_set_an_empty_step_memory_id() { + let transforms = |memory_id: &str| -> HashMap { + HashMap::from([( + "memory_id".to_string(), + serde_json::from_str(memory_id).unwrap(), + )]) + }; + let args = || -> AIAgentArgs { + serde_json::from_value(serde_json::json!({ + "provider": { "kind": "openai", "resource": {}, "model": "m" }, + "memory_id": null, + })) + .unwrap() + }; + for (transform, expected) in [ + (r#"{ "type": "static" }"#, None), + (r#"{ "type": "static", "value": "" }"#, None), + (r#"{ "type": "ai" }"#, None), + ( + r#"{ "type": "javascript", "expr": "flow_input.customer_id" }"#, + Some(""), + ), + ] { + let mut args = args(); + keep_authored_memory_id(&mut args, &transforms(transform)); + assert_eq!(args.memory_id.as_deref(), expected, "{transform}"); + } + } + + /// Only text output sends previous messages, so they never stand in for an image prompt. + #[test] + fn previous_messages_never_stand_in_for_an_image_prompt() { + let args: AIAgentArgs = serde_json::from_value(serde_json::json!({ + "provider": { "kind": "openai", "resource": {}, "model": "m" }, + "previous_messages": [{ "role": "user", "content": "earlier" }], + })) + .unwrap(); + let (history, _) = resolve_history_source(&args, None, "ws", "f/flow"); + assert!(has_prompt(&history, false, true, false)); + assert!(!has_prompt(&history, false, false, false)); + assert!(has_prompt(&history, true, false, false)); + assert!(!has_prompt( + &HistorySource::Messages(&[]), + false, + true, + false + )); + // A legacy `manual` memory ran on an empty list alone, and still does for text output. + assert!(has_prompt(&HistorySource::Messages(&[]), false, true, true)); + assert!(!has_prompt( + &HistorySource::Messages(&[]), + false, + false, + true + )); + } + + #[test] + fn reasoning_keeps_every_iteration_in_order() { + let mut acc = String::new(); + append_reasoning(&mut acc, Some("I need both cities.\n\n")); + append_reasoning(&mut acc, None); + append_reasoning(&mut acc, Some(" ")); + append_reasoning(&mut acc, Some("Paris is closer.")); + assert_eq!(acc, "I need both cities.\n\nParis is closer."); + } + + #[test] + fn an_unwritten_streaming_field_streams() { + assert!(streaming_requested(None)); + assert!(streaming_requested(Some(true))); + assert!(!streaming_requested(Some(false))); + } + + #[test] + fn max_iterations_partial_result_keeps_the_action_tags() { + let messages = vec![OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text("{\"rows\":2}".to_string())), + tool_call_id: Some("call_1".to_string()), + agent_action: Some(AgentAction::ToolCall { + job_id: uuid::Uuid::nil(), + function_name: "list_payouts".to_string(), + module_id: "b".to_string(), + }), + ..Default::default() + }]; + + let partial = MaxIterPartialResult { + messages: messages + .iter() + .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) + .collect(), + }; + let json = serde_json::to_value(&partial).unwrap(); + + let action = &json["messages"][0]["agent_action"]; + assert_eq!(action["type"], "tool_call"); + assert_eq!(action["function_name"], "list_payouts"); + } + /// Over 64 characters OpenAI rejects the key outright, which costs a wasted round /// trip per run and silently leaves that step with no prompt caching at all. #[test] @@ -1827,6 +2476,123 @@ mod tests { assert!(matches!(&tools[2].value, ToolValue::Mcp(_))); } + #[test] + fn narrow_roster_keeps_the_entries_a_run_named() { + fn named(id: &str, summary: &str) -> AgentTool { + AgentTool { + id: id.to_string(), + summary: Some(summary.to_string()), + description: None, + value: ToolValue::FlowModule(FlowModuleValue::Script { + input_transforms: HashMap::new(), + path: "u/test/tool".to_string(), + hash: None, + tag_override: None, + is_trigger: None, + pass_flow_input_directly: None, + }), + } + } + fn mcp(id: &str, summary: &str, path: &str) -> AgentTool { + AgentTool { + id: id.to_string(), + summary: Some(summary.to_string()), + description: None, + value: ToolValue::Mcp(windmill_common::flows::McpToolValue { + resource_path: path.to_string(), + include_tools: vec![], + exclude_tools: vec![], + }), + } + } + fn websearch(id: &str, summary: Option<&str>) -> AgentTool { + AgentTool { + id: id.to_string(), + summary: summary.map(str::to_string), + description: None, + value: ToolValue::Websearch(windmill_common::flows::WebsearchToolValue {}), + } + } + let roster = || { + vec![ + named("a", "get_user"), + named("b", "send_email"), + mcp("c", "github", "$res:u/test/gh"), + ] + }; + let names = |tools: &[AgentTool]| -> Vec { + tools.iter().filter_map(|t| t.summary.clone()).collect() + }; + let ids = + |tools: &[AgentTool]| -> Vec { tools.iter().map(|t| t.id.clone()).collect() }; + + // No list at all: the whole roster, as every agent written before the field expects. + assert_eq!( + names(&narrow_roster(roster(), None)), + ["get_user", "send_email", "github"] + ); + + // An empty list is a list: nothing is advertised, and no server is resolved to find that + // out — one that is down must not fail a run that switched it off. + assert!(names(&narrow_roster(roster(), Some(&[]))).is_empty()); + + let enabled = ["get_user".to_string(), "renamed_away".to_string()]; + assert_eq!( + names(&narrow_roster(roster(), Some(&enabled))), + ["get_user"] + ); + // Counted, not quoted: a name is an argument value, and one holding `$var:` arrives as the + // variable's own value, which this log is not masked for. + assert_eq!( + unmatched_enabled_tools_message(&enabled, &["get_user", "send_email", "u/test/gh"]) + .unwrap(), + "--- ENABLED TOOLS: 1 name named no tool of this agent and had no effect ---\n" + ); + + // A server is named by the resource it points at, bare, and never by its summary: that + // label is shown to nobody and two entries may carry the same one. + let named_server = ["u/test/gh".to_string()]; + assert_eq!( + names(&narrow_roster(roster(), Some(&named_server))), + ["github"] + ); + assert!(unmatched_enabled_tools_message(&named_server, &["u/test/gh"]).is_none()); + // Alongside a name that does match, so the summary being rejected is what empties it. + let summary_and_tool = ["get_user".to_string(), "github".to_string()]; + assert_eq!( + names(&narrow_roster(roster(), Some(&summary_and_tool))), + ["get_user"] + ); + assert!(narrow_roster(roster(), Some(&["u/test/other".to_string()])).is_empty()); + + // Web search reaches the model as a provider capability rather than a tool, so it has no + // name of its own and is enabled by a reserved one, whatever label it was authored with. + for label in [None, Some("Web Search")] { + let mut with_websearch = roster(); + with_websearch.push(websearch("w", label)); + assert_eq!( + ids(&narrow_roster( + with_websearch, + Some(&[WEBSEARCH_ENABLED_NAME.to_string()]) + )), + ["w"] + ); + } + } + + #[test] + fn a_tool_cannot_take_the_name_web_search_is_enabled_by() { + // Nothing else in a roster may answer to the reserved name, or enabling that tool would + // switch web search on beside it. Held here rather than by the shape of the name, which is + // an ordinary identifier: the run refuses to start instead. + assert!(TOOL_NAME_REGEX.is_match(WEBSEARCH_ENABLED_NAME)); + assert!(flow_module_tool_name(Some(WEBSEARCH_ENABLED_NAME)).is_err()); + + assert_eq!(flow_module_tool_name(Some("get_user")).unwrap(), "get_user"); + assert!(flow_module_tool_name(Some("get user")).is_err()); + assert!(flow_module_tool_name(None).is_err()); + } + #[test] fn tool_description_prefers_explicit_over_derived_and_name() { assert_eq!( diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index b3324f87ba..b577097c9d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -408,16 +408,6 @@ pub fn create_empty_dir(path: &PathBuf) -> std::io::Result<()> { } } -/// Signals a detached `spawn_blocking` task that the future awaiting it is -/// gone, so it can stop instead of running to completion in the background. -struct AbortOnDrop(std::sync::Arc); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); - } -} - /// Lay down the tree of an app-backed repository, which git can't clone /// because its URL carries no credential. /// @@ -490,7 +480,7 @@ async fn fetch_repo_archive( // stopping it, so the flag is what a cancelled job uses to reach the // extraction loop. The guard sets it when this future is dropped. let aborted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _abort_on_drop = AbortOnDrop(aborted.clone()); + let _abort_on_drop = crate::common::AbortOnDrop(aborted.clone()); let unpack_archive = download_archive.clone(); tokio::task::spawn_blocking(move || { unpack_repo_archive(&unpack_archive, &download_target, &aborted) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1fb566e57f..117fbda086 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; use serde_json::value::RawValue; use uuid::Uuid; -use windmill_parser_ts::remove_pinned_imports; +use windmill_parser_ts::{remove_pinned_import_specifiers, remove_pinned_imports}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo}; @@ -868,7 +868,7 @@ pub async fn install_bun_lockfile( if quiet { Some(&mut quiet_buf) } else { None }, None, ) - .warn_after_seconds(10) + .warn_after_seconds_for(10, "bun install") .await; if quiet && result.is_err() { // On failure, flush suppressed install output so the user can diagnose @@ -1131,9 +1131,12 @@ pub async fn generate_bun_bundle( None, None, ) + .warn_after_seconds_for(60, "bun build") .await?; } else { - let output = Box::into_pin(child_process.wait_with_output()).await?; + let output = Box::into_pin(child_process.wait_with_output()) + .warn_after_seconds_for(60, "bun build") + .await?; if !output.status.success() { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); @@ -1146,6 +1149,81 @@ pub async fn generate_bun_bundle( Ok(()) } +/// [`generate_bun_bundle`], built once more with the version pins dropped from the import +/// specifiers of `main.ts` if it fails. The lockfile pins those versions, but bun fails on a +/// pinned specifier except where it tolerates a failed import (in a `try`, under a `.catch`, in +/// dead code). Such a script builds as written and must keep that bundle, so only failures retry. +async fn generate_bun_bundle_unpinning_imports( + job_dir: &str, + w_id: &str, + job_id: &Uuid, + worker_name: &str, + db: Option<&Connection>, + timeout: Option, + mem_peak: &mut i32, + canceled_by: &mut Option, + common_bun_proc_envs: &HashMap, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, +) -> Result<()> { + let built = generate_bun_bundle( + job_dir, + w_id, + job_id, + worker_name, + db, + timeout, + mem_peak, + canceled_by, + common_bun_proc_envs, + occupancy_metrics, + ) + .await; + // Without a job, a failed build comes back as an `ExecutionErr`; with one, that variant is a + // cancellation or timeout, which must not be retried. + let build_failed = match &built { + Err(error::Error::ExitStatus(..)) => true, + Err(_) => db.is_none(), + Ok(()) => false, + }; + if !build_failed { + return built; + } + let Some(unpinned) = read_file_content(&format!("{job_dir}/main.ts")) + .await + .ok() + .and_then(|main| { + remove_pinned_import_specifiers(&main) + .ok() + .filter(|u| *u != main) + }) + else { + return built; + }; + write_file(job_dir, "main.ts", &unpinned)?; + if let Some(db) = db { + append_logs( + job_id, + w_id, + "\nbundling again with the imports' versions taken from the lockfile\n", + db, + ) + .await; + } + generate_bun_bundle( + job_dir, + w_id, + job_id, + worker_name, + db, + timeout, + mem_peak, + canceled_by, + common_bun_proc_envs, + occupancy_metrics, + ) + .await +} + struct PulledCodebase { is_esm: bool, } @@ -1203,7 +1281,12 @@ async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result, temp_script_refs: &Option>, + modules: Option<&HashMap>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path( inner_content, @@ -1264,6 +1348,7 @@ pub async fn prebundle_bun_script( db, w_id, temp_script_refs, + modules, ) .await; if exists_in_cache(&local_path, &remote_path).await { @@ -1303,7 +1388,7 @@ pub async fn prebundle_bun_script( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; - generate_bun_bundle( + generate_bun_bundle_unpinning_imports( job_dir, w_id, job_id, @@ -1319,7 +1404,9 @@ pub async fn prebundle_bun_script( ensure_bundle_output_exists(&origin)?; - save_cache(&local_path, &remote_path, &origin, false).await?; + save_cache(&local_path, &remote_path, &origin, false) + .warn_after_seconds_for(60, "bundle cache save") + .await?; Ok(()) } @@ -1442,6 +1529,7 @@ pub async fn compute_bundle_local_and_remote_path( db: Option<&DB>, w_id: &str, temp_script_refs: &Option>, + modules: Option<&HashMap>, ) -> (String, String) { let mut input_src = format!("{inner_content}{lock}",); @@ -1470,7 +1558,10 @@ pub async fn compute_bundle_local_and_remote_path( let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await; input_src.push_str(&ws_suffix); - let hash = windmill_common::utils::calculate_hash(&input_src); + + // The loader resolves relative imports against the module files in the job dir, so + // their content is inlined into the bundle this name covers. + let hash = crate::worker::artifact_cache_name(input_src, modules); let local_path = format!("{}/{hash}", *BUN_BUNDLE_CACHE_DIR); #[cfg(windows)] @@ -1569,6 +1660,7 @@ pub async fn handle_bun_job( Some(db), &job.workspace_id, &temp_script_refs, + modules.as_ref(), ) .await } @@ -1585,7 +1677,9 @@ pub async fn handle_bun_job( } }; - let (cache, logs) = crate::global_cache::load_cache(&local_path, &remote_path, false).await; + let (cache, logs) = crate::global_cache::load_cache(&local_path, &remote_path, false) + .warn_after_seconds_for(60, "bundle cache load") + .await; (cache, logs, local_path, remote_path) } else { (false, "".to_string(), "".to_string(), "".to_string()) @@ -1777,8 +1871,19 @@ pub async fn handle_bun_job( if modules.as_ref().is_some_and(|m| !m.is_empty()) { let bundle_path = std::path::Path::new(job_dir).join("out").join("main.js"); if bundle_path.exists() { + // The lock-generation build kept every `pkg@version` specifier, and bun resolves + // a pinned specifier outside node_modules, loading a second copy of the package. + // The bundle holds the user's code too, so only the specifiers are rewritten, and + // a bundle the parser rejects still runs as built, pins and all. let bundled = std::fs::read_to_string(&bundle_path)?; - write_file(job_dir, "main.ts", &bundled)?; + let unpinned = remove_pinned_import_specifiers(&bundled).unwrap_or_else(|e| { + tracing::warn!( + job_id = %job.id, + "could not unpin the modules bundle, running it as built: {e:#}" + ); + bundled + }); + write_file(job_dir, "main.ts", &unpinned)?; } } "\n\n--- BUN CODE EXECUTION ---\n".to_string() @@ -1892,8 +1997,12 @@ pub async fn handle_bun_job( // Kept comment-free — this string is written out per job. // `_takePendingStepFailure` / `_takePendingSuspend` hand back what the body - // caught and swallowed; honour them instead of reporting a `complete` (see - // `_pendingStepFailure` in client.ts). Optional: npm clients may predate them. + // caught and swallowed; honour them instead of reporting a bare `complete` + // (see client.ts). Optional: npm clients may predate them. + // `_warnUnobservedTaskFailures` reports what the body never awaited, and so + // belongs only on the paths that end the round for good. A round that + // dispatches, sleeps, checkpoints or waits for approval replays later and + // re-registers the same failures from the checkpoint — keep those quiet. let wrapper_content = if is_wac_v2 { format!( r#" @@ -1951,6 +2060,7 @@ async function run() {{ if (trailing.length > 0) {{ return {{ type: "dispatch", mode: trailing.length > 1 ? "parallel" : "sequential", steps: trailing }}; }} + ctx._warnUnobservedTaskFailures?.(); return {{ type: "complete", result: result ?? null }}; }} catch (e) {{ setWorkflowCtx(null); @@ -1963,13 +2073,14 @@ async function run() {{ return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null, started_at: dispatch.started_at, duration_ms: dispatch.duration_ms }}; }} if (dispatch.mode === "approval") {{ - return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled }}; + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled, skin: dispatch.skin, description: dispatch.description }}; }} if (dispatch.mode === "sleep") {{ return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; }} return {{ type: "dispatch", mode: dispatch.mode ?? "sequential", steps: dispatch.steps ?? [] }}; }} + ctx._warnUnobservedTaskFailures?.(); const failed = ctx._takePendingStepFailure?.(); if (failed) {{ throw failed.error; @@ -2178,7 +2289,7 @@ try {{ if !codebase.is_some() && !has_bundle_cache { if build_cache { - generate_bun_bundle( + generate_bun_bundle_unpinning_imports( job_dir, &job.workspace_id, &job.id, @@ -2194,7 +2305,10 @@ try {{ let bundle_path = format!("{job_dir}/main.js"); ensure_bundle_output_exists(&bundle_path)?; if !local_path.is_empty() { - match save_cache(&local_path, &remote_path, &bundle_path, false).await { + match save_cache(&local_path, &remote_path, &bundle_path, false) + .warn_after_seconds_for(60, "bundle cache save") + .await + { Err(e) => { let em = format!("could not save {local_path} to bundle cache: {e:?}"); tracing::error!(em) @@ -2565,7 +2679,8 @@ try {{ // WAC v2 post-execution: parse output and handle dispatch/suspend if is_wac_v2 { - return handle_wac_v2_output(result, job, conn, modules, new_args.as_ref()).await; + return handle_wac_v2_output(result, job, conn, canceled_by, modules, new_args.as_ref()) + .await; } Ok(result) @@ -2595,11 +2710,13 @@ pub async fn handle_wac_v2_output( result: Box, job: &MiniPulledJob, conn: &Connection, + canceled_by: &mut Option, modules: &Option>, preprocessed_args: Option<&HashMap>>, ) -> error::Result> { use crate::wac_executor::{ - load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, WacOutput, + load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, + wac_cancelled_mid_segment, WacOutput, WacPark, }; use serde_json::Value; use windmill_common::get_latest_flow_version_info_for_path; @@ -2757,15 +2874,17 @@ pub async fn handle_wac_v2_output( .collect(); // Resolve job_payload once (same for all children since they re-run - // the parent script) + // the parent script). The step's cache setting is for the workflow's + // result; a task is cached only through its own `cache_ttl` option, + // under a key of its own (see `cached_result_path`). let job_payload_template = match job.kind { JobKind::Script => { if let Some(hash) = job.runnable_id { Ok(JobPayload::ScriptHash { hash, path: job.runnable_path.clone().unwrap_or_default(), - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), priority: job.priority, @@ -2780,6 +2899,27 @@ pub async fn handle_wac_v2_output( )) } } + // A deployed flow runs an inline step as the `flow_node` its deploy + // rewrote it into; the child re-runs that node the way a `Script` + // child re-runs its hash, so `runnable_id` (the checkpoint's source + // hash) stays the same across parent and children. + JobKind::FlowScript => { + if let Some(id) = job.runnable_id { + Ok(JobPayload::FlowScript { + id: windmill_common::flows::FlowNodeId(id.0), + path: job.runnable_path.clone().unwrap_or_default(), + language: job.script_lang.unwrap_or(ScriptLang::Bun), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: ConcurrencySettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 FlowScript job missing runnable_id".to_string(), + )) + } + } JobKind::Preview => { let row: Option<(Option, Option)> = sqlx::query_as( "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", @@ -2795,8 +2935,8 @@ pub async fn handle_wac_v2_output( hash: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), lock: lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -2812,9 +2952,30 @@ pub async fn handle_wac_v2_output( // Step 1: Save checkpoint, suspend parent, and seed child checkpoints // in a single transaction — all BEFORE children become visible. + let segment_ms; { let mut tx = db.begin().await?; + // Park before writing the checkpoint. This locks the queue row ahead of the + // status row, the order `record_child_completion` takes, so a stale child + // finishing while the parent re-dispatches cannot deadlock this transaction. + // A cancel already on the row is also seen before anything is written, and + // every child pushed after the commit finds a parked parent to decrement. + match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + num_steps as i32, + 14.0 * 24.0 * 3600.0, + ) + .await? + { + WacPark::Parked(ms) => segment_ms = ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + } + // Update checkpoint with pending steps update_checkpoint_for_dispatch(&mut checkpoint, &steps, &mode, &job_ids); let status_json = serde_json::to_value(&checkpoint).map_err(|e| { @@ -2864,25 +3025,6 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent before children become visible. - // Keep running = true so the normal pull query ignores it. - // The suspended pull query picks it up when suspend reaches 0 - // (it checks: suspend_until IS NOT NULL AND suspend <= 0). - let suspend_count = num_steps as i32; - sqlx::query!( - "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", - job.id, - suspend_count, - ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Failed to suspend WAC parent job {}: {e}", - job.id - )) - })?; - tx.commit().await?; } @@ -2893,6 +3035,12 @@ pub async fn handle_wac_v2_output( let mut pushed_ids: Vec = Vec::with_capacity(num_steps); let push_result: error::Result<()> = async { for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // A task with a runnable of its own (a deployed script or flow) queues + // at that runnable's priority; any other task is the parent's code and + // queues at the parent's. + let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow") + && !step.script.starts_with("./"); + // Resolve job payload based on dispatch_type let (job_payload, child_args, is_external, on_behalf_of) = match step.dispatch_type.as_str() { @@ -2906,8 +3054,8 @@ pub async fn handle_wac_v2_output( hash: None, language: module.language, lock: module.lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -2962,9 +3110,8 @@ pub async fn handle_wac_v2_output( version: flow_info.version, labels: flow_info.labels.clone(), }; - let on_behalf_of = flow_info - .on_behalf_of(&job.workspace_id, db) - .await?; + let on_behalf_of = + flow_info.on_behalf_of(&job.workspace_id, db).await?; let step_args: HashMap> = step .args .iter() @@ -2992,7 +3139,8 @@ pub async fn handle_wac_v2_output( let mut job_payload = job_payload; if let Some(cache_ttl) = step.cache_ttl { match &mut job_payload { - JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } + | JobPayload::FlowScript { cache_ttl: ref mut ct, .. } => { *ct = Some(cache_ttl) } JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), @@ -3004,7 +3152,8 @@ pub async fn handle_wac_v2_output( || step.concurrency_time_window_s.is_some() { match &mut job_payload { - JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } + | JobPayload::FlowScript { concurrency_settings: ref mut cs, .. } => { if let Some(limit) = step.concurrent_limit { cs.concurrent_limit = Some(limit); } @@ -3070,13 +3219,14 @@ pub async fn handle_wac_v2_output( job.visible_to_owner, step.tag.clone().or_else(|| Some(job.tag.clone())), step.timeout.or(job.timeout), - None, // flow_step_id - step.priority, // priority_override - None, // authed - false, // running - None, // end_user_email - None, // trigger - None, // suspended_mode + None, // flow_step_id + step.priority + .or(if own_runnable { None } else { job.priority }), + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode ) .await?; @@ -3161,10 +3311,17 @@ pub async fn handle_wac_v2_output( .execute(db) .await; - // Unsuspend parent so the error propagates instead of a 14-day hang + // Unsuspend parent so the error propagates instead of a 14-day hang. + // Unlike the other suspend exits this one completes the job for real, so + // it needs its segment start back — the in-memory copy is what the pull + // stamped, before the suspend cleared the column. let _ = sqlx::query!( - "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + "UPDATE v2_job_queue + SET suspend = 0, suspend_until = NULL, + started_at = coalesce(started_at, $2, now()) + WHERE id = $1", job.id, + job.started_at, ) .execute(db) .await; @@ -3177,12 +3334,13 @@ pub async fn handle_wac_v2_output( "WAC v2 parent job suspended" ); + crate::wac_executor::end_wac_segment(conn, job, segment_ms); Err(error::Error::WacSuspended(format!( "WAC v2 job {} suspended waiting for {} child job(s)", job.id, num_steps ))) } - WacOutput::Approval { key, timeout, form, self_approval_disabled } => { + WacOutput::Approval { key, timeout, form, self_approval_disabled, skin, description } => { let db = match conn { Connection::Sql(db) => db, _ => { @@ -3204,6 +3362,23 @@ pub async fn handle_wac_v2_output( let mut tx = db.begin().await?; + // Park first: the queue row is locked before the status row, the order every + // child completion takes. + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, + timeout_secs, + ) + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; + // Save checkpoint let status_json = serde_json::to_value(&checkpoint).map_err(|e| { error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) @@ -3298,15 +3473,19 @@ pub async fn handle_wac_v2_output( }; // Store approval form metadata for the approval page endpoint - let approval_meta = serde_json::json!({ + let mut approval_meta = serde_json::json!({ "key": key, "form": form, "timeout": timeout_secs as u32, "self_approval_disabled": sad, + "skin": skin.unwrap_or_default(), "resume": resume_url, "cancel": cancel_url, "approvalPage": approval_page_url, }); + if let Some(description) = description.filter(|d| !d.is_null()) { + approval_meta["description"] = description; + } sqlx::query( "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( COALESCE(workflow_as_code_status, '{}'::jsonb), @@ -3354,16 +3533,8 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent with suspend=1 (waiting for 1 approval event) - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", - job.id, - timeout_secs, - ) - .execute(&mut *tx) - .await?; - tx.commit().await?; + crate::wac_executor::end_wac_segment(conn, job, segment_ms); tracing::info!( job_id = %job.id, @@ -3399,6 +3570,24 @@ pub async fn handle_wac_v2_output( let mut tx = db.begin().await?; + // Park first: the queue row is locked before the status row, the order every + // child completion takes. suspend=1 (not 0) so the suspended pull query only + // picks it up when `suspend_until <= now()`, not via `suspend <= 0`. + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, + sleep_secs, + ) + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; + // Save checkpoint let status_json = serde_json::to_value(&checkpoint).map_err(|e| { error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) @@ -3447,18 +3636,8 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent — it will auto-resume when suspend_until passes. - // Use suspend=1 (not 0) so the suspended pull query only picks it up - // when `suspend_until <= now()`, not via `suspend <= 0`. - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", - job.id, - sleep_secs, - ) - .execute(&mut *tx) - .await?; - tx.commit().await?; + crate::wac_executor::end_wac_segment(conn, job, segment_ms); tracing::info!( job_id = %job.id, @@ -3494,6 +3673,28 @@ pub async fn handle_wac_v2_output( let source_hash = job.runnable_id.map(|h| h.0.to_string()); let mut tx = db.begin().await?; + // Reset running=false so the job is immediately eligible for pickup. + // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — + // the job should be re-run right away to continue past the cached step. + // `prev` holds the pre-update row: RETURNING would see the cleared column. + // Runs before the checkpoint write so the queue row is locked ahead of the + // status row, the order every child completion takes. + let segment_ms = sqlx::query_scalar!( + "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) + UPDATE v2_job_queue q SET running = false, started_at = null + FROM prev WHERE q.id = $1 + RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + job.id, + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to reset running state for inline checkpoint: {e}" + )) + })? + .flatten(); + crate::wac_executor::persist_inline_checkpoint_delta( &mut tx, &job.id, @@ -3505,22 +3706,8 @@ pub async fn handle_wac_v2_output( ) .await?; - // Reset running=false so the job is immediately eligible for pickup. - // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — - // the job should be re-run right away to continue past the cached step. - sqlx::query!( - "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", - job.id, - ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Failed to reset running state for inline checkpoint: {e}" - )) - })?; - tx.commit().await?; + crate::wac_executor::end_wac_segment(conn, job, segment_ms); Err(error::Error::WacSuspended(format!( "WAC v2 job {} inline checkpoint for step {}", @@ -4382,4 +4569,49 @@ export function main(x: number) { return x; }"#; assert!(wrapper.contains(r#"line.startsWith("exec_preprocess:")"#)); assert!(wrapper.contains(r#"line.startsWith("exec:")"#)); } + + /// The bundle cache is global and content-keyed, so a key that ignores the inline + /// modules hands one workspace's bundle — attacker helper code and all — to the next + /// job whose main content and lockfile happen to match. + #[tokio::test] + async fn bundle_cache_key_separates_inline_module_content() { + use windmill_common::scripts::ScriptModule; + + async fn key_for(modules: Option<&HashMap>) -> String { + compute_bundle_local_and_remote_path( + "import { h } from './helper.ts';\nexport async function main() { return h(); }", + "{}\n//bun.lock\n", + "u/alice/script", + None, + "w1", + &None, + modules, + ) + .await + .1 + } + fn modules(content: &str) -> HashMap { + HashMap::from([( + "helper.ts".to_string(), + ScriptModule { + content: content.to_string(), + language: ScriptLang::Bun, + lock: None, + }, + )]) + } + + let attacker = key_for(Some(&modules("export const h = () => 'attacker'"))).await; + let victim = key_for(Some(&modules("export const h = () => 'victim'"))).await; + assert_ne!(attacker, victim); + assert_eq!( + attacker, + key_for(Some(&modules("export const h = () => 'attacker'"))).await, + "same modules must still share a cache slot" + ); + + // An absent map and an empty one are the same script, so they share a slot. + assert_eq!(key_for(None).await, key_for(Some(&HashMap::new())).await); + assert_ne!(key_for(None).await, attacker); + } } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index f6391eab07..9c012fd117 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -67,6 +67,20 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Tells a `spawn_blocking` task to stop when the future awaiting it goes away. +/// +/// Dropping a `JoinHandle` detaches the task rather than cancelling it, so a +/// cancelled or timed-out phase otherwise leaves the blocking pool working on an +/// answer nobody will read. Hold one of these beside the handle and have the +/// blocking loop check the flag. +pub(crate) struct AbortOnDrop(pub(crate) std::sync::Arc); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::Relaxed); + } +} + /// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string /// becomes its inner value, anything else is re-serialized compactly. pub(crate) fn raw_to_string(x: &str) -> String { @@ -1545,7 +1559,7 @@ pub async fn cached_result_path( client: &AuthedClient, job: &MiniPulledJob, raw_data: Option<&RawData>, -) -> String { +) -> windmill_common::error::Result { let mut hasher = sha2::Sha256::new(); hasher.update(&[job.kind as u8]); if let Some(ScriptHash(hash)) = job.runnable_id { @@ -1560,6 +1574,13 @@ pub async fn cached_result_path( _ => {} } } + // A workflow-as-code task child runs its parent's code with the parent's + // arguments; the step it executes is what tells its result from the parent's + // and from its siblings'. + if let Some(step_key) = wac_executing_key(db, job).await? { + hasher.update(b"wac_step:"); + hasher.update(step_key.as_bytes()); + } hash_args( db, client, @@ -1570,7 +1591,26 @@ pub async fn cached_result_path( job.cache_ignore_s3_path.unwrap_or(false), ) .await; - format!("g/results/{:064x}", hasher.finalize()) + Ok(format!("g/results/{:064x}", hasher.finalize())) +} + +/// The checkpoint step key a workflow-as-code parent seeded for this child at push +/// time; `None` for any job that is not such a child. +async fn wac_executing_key( + db: &DB, + job: &MiniPulledJob, +) -> windmill_common::error::Result> { + if job.parent_job.is_none() || job.flow_step_id.is_some() { + return Ok(None); + } + let key: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->>'_executing_key' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(job.id) + .fetch_optional(db) + .await?; + Ok(key.flatten()) } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index c95107d504..d393171f7f 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{utils::calculate_hash, worker::write_file}; +use windmill_common::worker::write_file; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; @@ -72,13 +72,21 @@ const CSHARP_OBJECT_STORE_PREFIX: &str = /// Cache key of a C# build. The run path and the deploy-time prebuild must derive it the /// same way or the prebuilt binary is never found and gets rebuilt on first run. #[cfg(feature = "csharp")] -async fn csharp_cache_key(code: &str, requirements_o: Option<&str>, w_id: &str) -> String { - let mut hash = calculate_hash(&format!( +async fn csharp_cache_key( + code: &str, + requirements_o: Option<&str>, + w_id: &str, + modules: Option<&HashMap>, +) -> String { + // The SDK project globs every `.cs` under the job dir, so companion modules are + // compiled into the binary this key names and have to be part of it. + let base = format!( "{}{}{}", code, requirements_o.unwrap_or(""), DOTNET_TARGET_FRAMEWORK.as_str() - )); + ); + let mut hash = crate::worker::artifact_cache_name(base, modules); hash.push_str(&crate::workspace_registry_cache_suffix(w_id).await); hash } @@ -487,10 +495,11 @@ pub async fn prebuild_csharp_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> error::Result> { check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; - let hash = csharp_cache_key(code, Some(lock), &job.workspace_id).await; + let hash = csharp_cache_key(code, Some(lock), &job.workspace_id, modules).await; let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -549,6 +558,7 @@ pub async fn handle_csharp_job( _worker_name: &str, _envs: HashMap, _occupancy_metrics: &mut OccupancyMetrics, + _modules: Option<&HashMap>, ) -> Result, Error> { Err(anyhow!("C# is not available because the feature is not enabled").into()) } @@ -568,6 +578,7 @@ pub async fn handle_csharp_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; @@ -575,6 +586,7 @@ pub async fn handle_csharp_job( inner_content, requirements_o.map(|x| x.as_str()), &job.workspace_id, + modules, ) .await; let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); diff --git a/backend/windmill-worker/src/dbt_column_index.rs b/backend/windmill-worker/src/dbt_column_index.rs new file mode 100644 index 0000000000..64f148534b --- /dev/null +++ b/backend/windmill-worker/src/dbt_column_index.rs @@ -0,0 +1,602 @@ +//! Column-level lineage and real column schemas, from the engine's own static +//! analysis. +//! +//! `manifest.json` carries neither. What does is the parquet index an engine +//! writes under `dbt compile --static-analysis strict --write-index`: +//! `dbt.column_lineage.parquet` (column-to-column edges, each labelled `copy`, +//! `mod` or `scan`) and `dbt.node_columns.parquet` (every column of every node, +//! typed and ordered, rather than only the ones an author documented). +//! +//! Four properties shape everything here, all of them measured against the real +//! engines rather than assumed: +//! +//! - **Strict analysis rejects SQL the default accepts.** An unresolvable +//! identifier is an error under `strict` and compiles fine otherwise, so this +//! is a SEPARATE pass with its own `--target-path`, never a flag on the build, +//! and it is opt-in per project. +//! - **A failed pass still writes the index**, holding every edge of the models +//! that did analyze. So the artifact is read whatever the exit status. +//! - **The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 accepts +//! `--write-index`, declares the views over these two tables in its own +//! `views.sql`, and writes neither file; only Fusion does today. Nothing here +//! asks which engine it is beyond "has the flag" — a release that starts +//! writing them is picked up with no change. +//! - **An incremental model has two shapes, and one ingest holds one of them.** +//! `is_incremental()` is false when the target does not exist or the build +//! is `--full-refresh`, so the `{{ this }}` self-join — and any `ref()` inside +//! that branch — compiles only in the other case. What this stores is +//! therefore what the compile in front of it saw: at DEPLOY, before the first +//! build, that is the cold shape, and a project deployed again after its +//! tables exist stores the incremental one for the same source. Nothing here +//! can reconcile that; dbt has no mode that emits both. The flag is taken from +//! the build so a per-run ingest matches its own run, and the version's graph +//! is honest about the compile that produced it rather than about every run +//! that will follow. + +use std::collections::HashSet; +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::path::Path; +use std::time::Duration; + +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::record::{Field, Row}; +use uuid::Uuid; +use windmill_common::dbt_manifest::{ + is_direct, ColumnIndex, IndexedColumn, IngestedColumnEdge, MAX_COLUMN_EDGES, +}; +use windmill_common::error; +use windmill_common::worker::Connection; +use windmill_parser_yaml::dbt::DbtDescriptor; +use windmill_queue::append_logs; + +use crate::dbt_executor::{dbt_command, Invocation, PreparedProject}; +use crate::handle_child::JobCtx; + +/// Where the lineage pass writes, relative to the project directory. +/// +/// Its own tree, not the runtime's `wm_target`: a `dbt compile` writes +/// `manifest.json` and `run_results.json` like any other invocation, and after a +/// build those two are what the graph ingest and `dbt retry` read. +const CLL_ARTIFACTS_DIR: &str = "wm_target_cll"; + +const COLUMN_LINEAGE_PARQUET: &str = "dbt.column_lineage.parquet"; +const NODE_COLUMNS_PARQUET: &str = "dbt.node_columns.parquet"; + +/// Run the lineage pass and read what it produced. +/// +/// Two steps with deliberately different contracts, because conflating them is +/// what made a best-effort annotation able to fail the job it annotates: +/// +/// - [`compile_index`] runs a subprocess and owns the JOB's semantics. Only a +/// cancellation or the job's own deadline can `Err` out of it; a non-zero exit +/// and an over-long output are outcomes, not failures. +/// - [`read_index`] owns the ARTIFACT's semantics. Reading it never fails the +/// job on the artifact's account: an absent, unreadable or partial index is a +/// value, not an error. It runs UNDER the poller all the same, so the job can +/// still end the phase — a cancel, a completion or the phase timeout — which +/// is the job's semantics reaching in, not the artifact's reaching out. +/// +/// The phase budget wraps the compile alone, because it exists to leave the +/// BUILD its share of the clock and only the compile can spend that share +/// unboundedly. The decode's own end is the job's: the poller it runs under +/// stops it when the job stops. +pub(crate) async fn collect( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + // The dbt subcommand the job runs, which decides the effective + // `--full-refresh` — see `dbt_executor::full_refresh`. + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + kept: &HashSet<&str>, +) -> error::Result> { + if !descriptor.column_lineage { + return Ok(None); + } + if !p.engine.engine.writes_column_index() { + append_logs( + job_id, + w_id, + format!( + "\n`column_lineage` is set, but the {} engine has no `--write-index`: column \ + lineage needs an engine that does static analysis. The rest of the graph is \ + unaffected.\n", + p.engine.engine.as_str() + ), + conn, + ) + .await; + return Ok(None); + } + + let index_dir = p.project_dir.join(CLL_ARTIFACTS_DIR).join("index"); + let Some(compiled) = compile_index(p, descriptor, inv, command, ctx, job_id, w_id, conn).await? else { + return Ok(None); + }; + let coverage = Coverage::of(&compiled); + + // Decoded UNDER the poller, not followed by a check of its own. The decode is + // the one phase of this pass with no subprocess behind it, so nothing else + // heartbeats while it runs: left alone, a large index is a silent worker for + // as long as it takes, which the zombie sweep reads as a dead job and + // restarts. The poller pings throughout and ends this with an `Err` if the + // job is cancelled or completed meanwhile — the job's own semantics, which + // this module may always propagate. + let artifact = crate::handle_child::run_future_with_polling_update_job_poller( + *job_id, + ctx.timeout(), + conn, + ctx.mem_peak, + ctx.canceled_by, + async { Ok(read_index(&index_dir, kept).await) }, + ctx.worker_name, + w_id, + &mut Some(ctx.occupancy_metrics), + Box::pin(futures::stream::empty()), + ) + .await?; + + // What only the pass knows. The COUNTS are logged where the index is folded + // into the graph, since the graph decides how much of it is kept. + let note = match artifact { + Artifact::Read(index) => { + if let Some(note) = coverage.caveat() { + log(job_id, w_id, note, &compiled.stderr, conn).await; + } + return Ok(Some(index)); + } + // The truncated arms come first: a compile stopped part-way explains an + // absent or unreadable artifact, and blaming the engine's capability + // for it sends the reader to check the wrong thing entirely. + Artifact::Missing if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass printed more than this runtime reads and was \ + stopped before it wrote `{COLUMN_LINEAGE_PARQUET}`." + ), + Artifact::Unreadable(why) if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass was stopped for printing more than this \ + runtime reads, and the `{COLUMN_LINEAGE_PARQUET}` it had written could not be read \ + ({why})." + ), + // Said apart from the one below, because it sends the reader somewhere + // else entirely: the engine did its job and this runtime could not read + // what it wrote. + Artifact::Unreadable(why) => format!( + "No column lineage: `{COLUMN_LINEAGE_PARQUET}` was written but could not be read \ + ({why}). The graph is unaffected." + ), + Artifact::Missing => format!( + "No column lineage: the analysis pass wrote no `{COLUMN_LINEAGE_PARQUET}`. Only an \ + engine that computes it does, and only for the warehouses it analyzes natively — \ + the flag alone is not the capability." + ), + }; + // The engine's own diagnostics come along. They are how a reader learns that + // this adapter turned static analysis off, which it reports as a warning on + // a SUCCESSFUL compile that nothing else would show. + log(job_id, w_id, ¬e, &compiled.stderr, conn).await; + Ok(None) +} + +async fn log(job_id: &Uuid, w_id: &str, note: &str, stderr: &str, conn: &Connection) { + append_logs( + job_id, + w_id, + format!("\n{note}\n{}", diagnostics(stderr)), + conn, + ) + .await; +} + +/// How completely the analysis compile covered the project. +/// +/// Every way the COMPILE can disappoint is a value here rather than an error. An +/// `Err` from `compile_index` is the JOB's — a cancellation or its deadline — +/// and must fail it; the pass giving up on its own terms is `Ok(None)` and has +/// already been logged. +enum Coverage { + /// Every model analyzed. + Whole, + /// `--static-analysis strict` rejected part of the project. Whatever it did + /// analyze is still in the index. + Partial, + /// The output ceiling killed the compile mid-run. Distinct from `Partial`: + /// nothing rejected the project, but the index is however far it had got, so + /// it is not `Whole` either. + Truncated, +} + +impl Coverage { + fn of(c: &crate::dbt_executor::Captured) -> Self { + match (c.truncated, c.success) { + (true, _) => Coverage::Truncated, + (false, true) => Coverage::Whole, + (false, false) => Coverage::Partial, + } + } + + /// What to tell the reader when an index WAS produced. `None` for a run that + /// covered everything, which needs no caveat. + fn caveat(&self) -> Option<&'static str> { + match self { + Coverage::Whole => None, + Coverage::Partial => Some( + "Column lineage: `--static-analysis strict` rejected part of the project, so \ + the lineage covers only the models it could analyze.", + ), + Coverage::Truncated => Some( + "Column lineage: the analysis pass printed more than this runtime reads and was \ + stopped, so the lineage covers only the models it had reached.", + ), + } + } +} + +/// Run `dbt compile --static-analysis strict --write-index`, under this phase's +/// share of the job's clock. +/// +/// `Ok(None)` is "the pass gave up and said so"; `Err` is the job's own +/// cancellation or deadline and must propagate. Nothing outlives this function: +/// the budget is a race around the child, and dropping that future kills it +/// through `run_captured`'s `kill_on_drop`. +async fn compile_index( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result> { + // A previous pass in the same job directory — a retry's second attempt — + // would otherwise be read back as this one's answer. + tokio::fs::remove_dir_all(p.project_dir.join(CLL_ARTIFACTS_DIR)) + .await + .ok(); + + // Static analysis can log in to read schemas, so it gets live credentials like + // every other dbt process. + if let Err(e) = p.refresh_profile(descriptor, job_id, w_id, conn).await { + append_logs( + job_id, + w_id, + format!("\nColumn lineage: skipped, {e}\n"), + conn, + ) + .await; + return Ok(None); + } + + let mut cmd = dbt_command( + p, + &[ + "compile", + "--static-analysis", + "strict", + "--write-index", + // Documented as what builds the CLL graph, and `--write-index` alone + // happens to imply it on the engine probed. Passed explicitly so the + // pass does not depend on which of the two is doing the work. + "--write-lineage", + "--target-path", + CLL_ARTIFACTS_DIR, + ], + ); + // The flag already wins over the env var dbt_command sets, but setting both + // means this pass cannot write into the runtime's artifacts even if that + // precedence ever changes — and what is in there after a build is the + // `run_results.json` a `dbt retry` resumes from. + cmd.env("DBT_TARGET_PATH", CLL_ARTIFACTS_DIR); + crate::dbt_executor::add_vars(&mut cmd, descriptor, inv)?; + // The BUILD's answer, not the descriptor's default: `is_incremental()` + // branches on it, so a model reading `{{ this }}` compiles its self-join — + // and any `ref()` inside that branch — only when this is absent. Guessing + // here stores lineage for SQL the run never executed. + if crate::dbt_executor::full_refresh(descriptor, inv, command)? { + cmd.arg("--full-refresh"); + } + // Captured rather than streamed: a strict-analysis failure is a wall of + // diagnostics about SQL the build itself accepts, and this pass decides + // nothing about whether that build runs. + // Read before the future below borrows `ctx` mutably. + let budget = phase_budget(ctx); + let run = crate::dbt_executor::run_captured( + cmd, + "dbt compile (column lineage)", + ctx, + job_id, + w_id, + conn, + CLL_MAX_OUTPUT_BYTES, + // The ceiling is this pass's, not the job's: a compile that prints more + // than it than we care to read has still analyzed the project, and the + // index it wrote is on disk either way. + crate::dbt_executor::Overflow::Truncate, + ); + let Some(budget) = budget else { + return Ok(Some(run.await?)); + }; + match tokio::time::timeout(budget, run).await { + Ok(r) => Ok(Some(r?)), + Err(_) => { + append_logs( + job_id, + w_id, + format!( + "\nNo column lineage: the analysis pass did not finish within {}s, half of \ + what was left of this job's time. The build below gets the rest.\n", + budget.as_secs() + ), + conn, + ) + .await; + Ok(None) + } + } +} + +/// stdout the pass may produce. It is a compile, so this is diagnostics rather +/// than data. +const CLL_MAX_OUTPUT_BYTES: usize = 1 << 20; + +/// The share of the job's remaining wall clock this pass may spend. +/// +/// A per-run refresh ingests BEFORE the build and shares the job's one deadline, +/// so an unbounded pass on a slow project would hand `dbt build` an expired +/// budget and fail the run it exists only to annotate. Half leaves the build at +/// least as long as the annotation was allowed to take. +/// +/// Spent as a race around the COMPILE rather than as a shortened deadline handed +/// to the runner: the runner reports its expiry as an `Err`, indistinguishable +/// from a cancellation or the job's own deadline, and those two MUST fail the +/// job. Expiring here is this budget and nothing else. The child dies with the +/// dropped future through `run_captured`'s `kill_on_drop`; the decode is outside +/// this race and answers to the poller instead. +fn phase_budget(ctx: &JobCtx<'_>) -> Option { + ctx.timeout() + .map(|left| Duration::from_secs((left.max(0) as u64 / 2).max(1))) +} + +/// The tail of what the engine said, bounded. The whole of it is every rendered +/// model on a large project, which is not what a job log is for. +const DIAGNOSTIC_LINES: usize = 40; + +fn diagnostics(out: &str) -> String { + let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect(); + let tail = &lines[lines.len().saturating_sub(DIAGNOSTIC_LINES)..]; + match tail.is_empty() { + true => String::new(), + false => format!("{}\n", tail.join("\n")), + } +} + +/// What came back from the artifact. Never an `Err`: nothing the file does or +/// fails to do is a reason to fail a job. `Unreadable` is separate from +/// `Missing` because the two send a reader looking in different places — one at +/// their engine and adapter, the other at a file that exists. +enum Artifact { + Read(ColumnIndex), + Missing, + Unreadable(String), +} + +/// Read both parquets, if the lineage one is there. +/// +/// The column schemas alone are not worth a graph: they arrive with the lineage +/// or not at all, and a node's declared columns already answer for the case +/// where the pass never ran. +async fn read_index(index_dir: &Path, kept: &HashSet<&str>) -> Artifact { + let lineage = index_dir.join(COLUMN_LINEAGE_PARQUET); + if !tokio::fs::try_exists(&lineage).await.unwrap_or(false) { + return Artifact::Missing; + } + let columns = index_dir.join(NODE_COLUMNS_PARQUET); + // Owned, because the decode moves to a blocking thread. The index describes + // the whole project while this graph describes one selection of it, so + // scoping HERE is what keeps the bound below from being spent on rows the + // graph would discard anyway. + let kept: HashSet = kept.iter().map(|s| (*s).to_string()).collect(); + // Dropping the handle of a blocking task does NOT stop it: the poller + // cancelling this phase would otherwise leave a thread decoding millions of + // rows for a job that is over. `abandoned` is set when this future is + // dropped, and the row loop reads it. + let abandoned = Arc::new(AtomicBool::new(false)); + let _stop = crate::common::AbortOnDrop(abandoned.clone()); + // Decompressing and decoding a parquet is CPU work on a file the engine just + // wrote, so it does not belong on the runtime's poll thread. + let read = tokio::task::spawn_blocking(move || { + read_index_blocking(&lineage, &columns, &kept, &abandoned) + }) + .await; + match read { + Ok(Ok(index)) => Artifact::Read(index), + Ok(Err(e)) => Artifact::Unreadable(e.to_string()), + Err(e) => Artifact::Unreadable(e.to_string()), + } +} + +fn read_index_blocking( + lineage: &Path, + columns: &Path, + kept: &HashSet, + abandoned: &AtomicBool, +) -> error::Result { + let mut out = ColumnIndex::default(); + // ONE pass, with the two kinds bucketed as they arrive. `copy` and `mod` say + // the value itself travelled, so they get the whole budget; `scan` — the + // column was read to produce the ROW, which reaches every output column of + // its model and is the bulk of a wide project's index — fills only what is + // left over at the end. Reading the file twice to get that ordering would + // double the decode of exactly the large index this bound exists for. + let mut scan: Vec = Vec::new(); + for_each_row(lineage, abandoned, |row| { + let lineage_kind = string(row, "lineage_kind"); + let parent_unique_id = string(row, "from_node_unique_id"); + let child_unique_id = string(row, "to_node_unique_id"); + let parent_column = string(row, "from_column_name"); + let child_column = string(row, "to_column_name"); + // A column of a node the analysis could not name is not an endpoint the + // graph can draw, and neither is one outside this graph's nodes. + if parent_column.is_empty() + || child_column.is_empty() + || !kept.contains(&parent_unique_id) + || !kept.contains(&child_unique_id) + { + return ControlFlow::Continue(()); + } + let edge = IngestedColumnEdge { + parent_unique_id, + parent_column, + child_unique_id, + child_column, + lineage_kind, + }; + // The bound covers BOTH buckets, so the pass never holds more than one + // budget's worth however the kinds are distributed. + let held = out.edges.len() + scan.len(); + if is_direct(&edge.lineage_kind) { + // A direct edge displaces a `scan` one: the budget is spent on + // value flow first. + if held >= MAX_COLUMN_EDGES { + scan.pop(); + } + out.edges.push(edge); + // The edge that FILLS the budget ends the read, not the next one to + // arrive: once the displacing kind is full nothing later in the file + // can be kept, and waiting for another direct edge to say so decodes + // a `scan`-only tail all the way to the backstop for nothing. + return match out.edges.len() >= MAX_COLUMN_EDGES { + true => ControlFlow::Break(()), + false => ControlFlow::Continue(()), + }; + } + if held < MAX_COLUMN_EDGES { + scan.push(edge); + } + // Not a stopping point even when full: a direct edge still to come takes + // a `scan` entry's place. + ControlFlow::Continue(()) + })?; + out.edges.append(&mut scan); + // Absent is normal — an engine can write the lineage table and not this one — + // and unreadable is not worth losing the lineage over. + let mut held = 0usize; + let _ = for_each_row(columns, abandoned, |row| { + let unique_id = string(row, "unique_id"); + let name = string(row, "column_name"); + if held >= MAX_INDEXED_COLUMNS { + return ControlFlow::Break(()); + } + if name.is_empty() || !kept.contains(&unique_id) { + return ControlFlow::Continue(()); + } + held += 1; + // The author's `data_type` where `schema.yml` gives one, since that is + // what the project calls the column; the analysis's own inference + // otherwise. + let column_type = match string(row, "declared_type") { + t if !t.is_empty() => t, + _ => string(row, "inferred_type"), + }; + out.columns + .entry(unique_id) + .or_default() + .push(IndexedColumn { + name, + column_type, + index: int(row, "column_index").unwrap_or(i64::MAX), + }); + ControlFlow::Continue(()) + }); + Ok(out) +} + +/// The most rows of `dbt.node_columns.parquet` one pass keeps. One per column of +/// the project, so the same bound as the edges is far more than any project +/// reaches; it exists for the same reason. +const MAX_INDEXED_COLUMNS: usize = MAX_COLUMN_EDGES; + +/// The most rows of an index one pass DECODES, whatever it keeps of them. +/// +/// A bound on work rather than on memory, and the two are separate because the +/// input this defends against is the one that cannot be collected: `scan` +/// lineage is emitted from every predicate and join column to every output +/// column, so a project shaped that way writes an index whose row count is +/// quadratic in its widest model. This pass runs outside the phase budget, on a +/// blocking thread, and nothing the file contains may fail a deploy or a run — +/// so the file it walks needs an end even when almost nothing in it is +/// retained. The abandonment flag ends it sooner when the job is over; this is +/// the bound for a job that is not. +const MAX_INDEX_ROWS: usize = 4_000_000; + +/// Decode a parquet a row at a time, handing each to `f` and never holding two. +/// +/// Collecting first would put a `Vec` — each row carrying its own copy of +/// every column NAME — in front of the caller's own bound, which is what would +/// take the worker process down on the index described above. +/// +/// `f` says when it has all it will take, and that is the ordinary end: this +/// runs outside the phase budget, so every row decoded past the point of being +/// able to keep one is wall clock the build below does not get. +fn for_each_row( + path: &Path, + abandoned: &AtomicBool, + mut f: impl FnMut(&Row) -> ControlFlow<()>, +) -> error::Result<()> { + let fail = |e: parquet::errors::ParquetError| { + error::Error::internal_err(format!("reading {}: {e}", path.display())) + }; + let file = std::fs::File::open(path) + .map_err(|e| error::Error::internal_err(format!("opening {}: {e}", path.display())))?; + let reader = SerializedFileReader::new(file).map_err(fail)?; + for (n, row) in reader.get_row_iter(None).map_err(fail)?.enumerate() { + // Nobody is waiting for this any more — the job was cancelled, completed + // or ran out of time while it decoded. + if abandoned.load(Ordering::Relaxed) { + break; + } + if n >= MAX_INDEX_ROWS { + tracing::warn!( + "dbt column index: {} holds more than {MAX_INDEX_ROWS} rows; the rest is dropped", + path.display() + ); + break; + } + if f(&row.map_err(fail)?).is_break() { + break; + } + } + Ok(()) +} + +/// By NAME, not by position: these tables are the engine's own schema and it +/// adds columns to them between releases. +fn field<'a>(row: &'a Row, name: &str) -> Option<&'a Field> { + row.get_column_iter() + .find(|(k, _)| k.as_str() == name) + .map(|(_, v)| v) +} + +fn string(row: &Row, name: &str) -> String { + match field(row, name) { + Some(Field::Str(s)) => s.clone(), + Some(Field::Bytes(b)) => String::from_utf8_lossy(b.data()).into_owned(), + _ => String::new(), + } +} + +fn int(row: &Row, name: &str) -> Option { + match field(row, name) { + Some(Field::Long(v)) => Some(*v), + Some(Field::Int(v)) => Some(*v as i64), + Some(Field::Short(v)) => Some(*v as i64), + Some(Field::UInt(v)) => Some(*v as i64), + Some(Field::ULong(v)) => i64::try_from(*v).ok(), + _ => None, + } +} diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 33ddd6c32e..8119897d6d 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -19,12 +19,13 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::client::AuthedClient; use windmill_common::error::{self, Error}; +use windmill_common::jobs::JobKind; use windmill_common::materialization::{ record_materialization, MaterializationStatus, RecordMaterializationRequest, }; use windmill_common::worker::{to_raw_value, write_file, Connection}; use windmill_parser_yaml::{ - parse_dbt_descriptor, DbtDescriptor, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, + parse_dbt_descriptor, DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, DBT_DEFAULT_WAREHOUSE, }; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -37,6 +38,9 @@ use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; use crate::dbt_profiles::{ ensure_adapter_licensed, render_dbt_profile, render_profile, DbtAdapter, KnownAdapter, }; +use crate::dbt_state::{ + environment_label, prepare_deferral, write_state_dir, Deferral, StateManifest, STATE_DIR, +}; use crate::handle_child::{ get_mem_peak, handle_child, run_future_with_polling_update_job_poller, JobCtx, JobDeadline, }; @@ -121,6 +125,12 @@ pub struct DbtRunResult { /// the same project — cannot get them from the job. #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub invocation_args: std::collections::HashMap>, + /// The run whose stored state this one resolved its unbuilt `ref()`s + /// through, absent when it deferred to none. What a deferring run built + /// against is otherwise unrecoverable: the state is replaced by the next + /// successful run of that environment. + #[serde(skip_serializing_if = "Option::is_none")] + pub deferred_to: Option, } #[derive(Serialize, Debug, Default)] @@ -189,7 +199,13 @@ pub(crate) async fn handle_dbt_job( // result publishes, and both describe an invocation of this script, not one // executor's view of it. let raw_args = job.args.as_ref().map(|a| a.0.clone()).unwrap_or_default(); - let inv = Invocation { args: args.clone(), raw_args, envs: envs.clone(), strict: true }; + let inv = Invocation { + args: args.clone(), + raw_args, + envs: envs.clone(), + deferral: None, + strict: true, + }; // One wall clock for the whole job. A dbt job is a sequence of // subprocesses — provision, deps, parse, ls, build, then the // `after_all` tests — and each would otherwise resolve the job's full @@ -262,6 +278,16 @@ pub(crate) async fn handle_dbt_job( // applies — nothing is built, so there is no test phase, no materialization, // no retry state and no ownership to publish. if command == "parse" { + // Checked here rather than at the seam below, which cannot tell a parse + // from a run that simply left `defer` off: a parse never reaches the + // deferral at all, so it is the one caller for which "turn `defer` on" + // would be advice that leads nowhere. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + StateAccess::Never(&command), + !selection_is_overridden(&descriptor, &inv.args)?, + )?; return run_parse_only( &prepared, &descriptor, @@ -364,6 +390,84 @@ pub(crate) async fn handle_dbt_job( inv }; + // Read AFTER the retry restore, so a retry defers exactly as the run it + // resumes did: a retry's own arguments are the command block alone, and the + // relations its unbuilt `ref()`s resolve to must not depend on that. + let defer = arg_bool(&inv.args, "defer")?.unwrap_or(descriptor.defer); + // Before the state is fetched, not only at the seam where the selection + // reaches dbt: a selector that cannot work whatever the state says would + // otherwise be masked by the "nothing published yet" refusal, which sends the + // caller to publish a state that will not help. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + if defer { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(&descriptor, &inv.args)?, + )?; + // A `show` defers too, and every engine takes the flags on it: it COMPILES + // the model it previews, so a model whose upstream this environment built and + // this run did not is exactly the case a deferral exists for. + let inv = if defer { + // Refused before anything runs. `dbt retry` reads the run it resumes + // from `--state`, the flag a deferral needs, so an engine without + // `--defer-state` can be given one or the other: told to defer, it + // resumes the stored state's own (successful) results and rebuilds + // nothing, and left alone it rebuilds the failed nodes with every + // `ref()` resolving into the schema THIS run writes — which for the + // narrowed run a deferral exists to serve is not where those models go. + if command == "retry" && !prepared.engine.engine.has_defer_state_flag() { + return Err(Error::BadRequest(format!( + "`{}` cannot resume a run that deferred: `dbt retry` takes the run it resumes \ + from `--state`, which is also where a deferral reads its manifest, and this \ + engine has no `--defer-state` to tell the two apart. Run the script again \ + instead of resuming it, or move the project to dbt-core-1x", + prepared.engine.engine.as_str() + ))); + } + let deferral = prepare_deferral(&prepared, &job.workspace_id, job_dir, conn).await?; + // Only answerable once the state is loaded: `defer` is enough for a + // `state:` method, which reads the manifest every publication carries, + // but a `result:` one reads `run_results.json` — and a build recovered by + // node retry publishes without it, since the results it holds describe + // only the nodes the retry rebuilt. dbt-core then raises an INTERNAL + // error and the Rust engines match nothing and exit 0. + if !deferral.has_run_results + && selection_names( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + &["result"], + ) + { + return Err(Error::BadRequest(format!( + "a `result:` selector reads `run_results.json` out of the published state, and \ + the state for this environment ({}) carries only the manifest run {} \ + published: a build recovered by node retry stores none, its results describing \ + the retried nodes rather than the whole build. Run this script once without \ + `defer` and without overrides to publish a complete state, or drop the selector", + environment_label(&prepared), + deferral.published_by + ))); + } + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nDeferring unbuilt refs to the dbt state published by run {}; this run \ + publishes none of its own\n", + deferral.published_by + ), + conn, + ) + .await; + Invocation { deferral: Some(deferral), ..inv } + } else { + inv + }; + // Ingested BEFORE the build, from a `dbt parse` with this run's vars, so the // models shown are the ones about to be built. Rows are keyed by path, version // AND job so no two runs collide; the path-keyed `asset` usage belongs to one @@ -387,7 +491,7 @@ pub(crate) async fn handle_dbt_job( // For a retry the restored manifest already describes the invocation // being resumed, so only the ingest runs — with that invocation's // arguments, which the selection resolver needs to interpolate. - ingest_from_run(&prepared, &descriptor, &inv, &mut ctx, job, conn).await?; + ingest_from_run(&prepared, &descriptor, &inv, &command, &mut ctx, job, conn).await?; } // A read-only command prints rows to stdout, so it is captured rather than @@ -428,9 +532,28 @@ pub(crate) async fn handle_dbt_job( // previous attempt's `run_results.json` is still in the job directory. Never on // an agent worker, which cannot read `v2_job_queue` — the wait below would be // uninterruptible, so a cancelled job would hold its slot and then start dbt. + // And never where the engine cannot be told to defer on a `retry`: the + // rebuild would resolve this run's unbuilt refs into the schema it writes + // into, so the nodes it "recovered" would read from the wrong relations. + // Said out loud below rather than silently skipped. + let retry_would_lose_the_deferral = + inv.deferral.is_some() && !prepared.engine.engine.has_defer_state_flag(); let node_retry = descriptor .retry_failed_nodes - .filter(|_| matches!(conn, Connection::Sql(_))); + .filter(|_| matches!(conn, Connection::Sql(_))) + .filter(|_| !retry_would_lose_the_deferral); + if descriptor.retry_failed_nodes.is_some() && retry_would_lose_the_deferral { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nSkipping the automatic node retry: `{}` cannot defer on a `dbt retry`\n", + prepared.engine.engine.as_str() + ), + conn, + ) + .await; + } let mut retries_left = node_retry.map(|p| p.attempts()).unwrap_or(0); if let Some(policy) = node_retry.filter(|_| run.is_err()) { retry_failed_nodes( @@ -519,6 +642,56 @@ pub(crate) async fn handle_dbt_job( { tracing::warn!("dbt: could not save retry state for job {}: {e:#}", job.id); } + // What a later run defers to, published by the runs whose relations are the + // SCRIPT's — the same condition that decides whether a run's graph becomes + // what the script owns, and for the same reason: an invocation that scoped + // its own models has no standing to say where this project's relations live. + // Success is the other half, because a relation a deferral resolves to has + // to exist. A `retry` is excluded: its `run_results.json` names only the + // nodes it redid, so publishing it would leave the environment claiming a + // run of a handful of models. + // + // And never a run that DEFERRED, whatever narrowed it. A deferring run built + // some of the relations its manifest names and resolved the rest out of the + // state it read, so publishing that manifest would record relations nothing + // built — and a model renamed since would be recorded under a name only a + // full build creates, breaking every later deferral until one repairs it. + // `publishes_ownership` cannot see this on its own: it reads the caller's + // overrides, and a descriptor that already narrows `select` needs none. + if run.is_ok() + && command == "build" + && inv.deferral.is_none() + // A run of the DEPLOYED version, by kind. A preview carries a + // caller-supplied `script_hash` into `runnable_id` + // (`run_preview_script`), so the version guard alone would let anyone who + // may run a job publish arbitrary content as a deployed script's state. + && job.kind == JobKind::Script + && prepared.graph_refresh.publishes_ownership() + { + // Losing it costs the next deferral, not the run that just finished — + // but silently, so the one actionable case (an artifact too large for + // the database on an instance with no object storage) says so. + if let Err(e) = crate::dbt_state::publish( + &prepared, + &job.workspace_id, + &job.id, + job.runnable_id.map(|h| h.0), + // An attempt was spent, so `run_results.json` on disk is the one + // `dbt retry` left: the nodes it redid, not the build. + node_retry.is_some_and(|p| retries_left < p.attempts()), + conn, + ) + .await + { + append_logs( + &job.id, + &job.workspace_id, + format!("\nCould not publish this run as the environment's dbt state: {e}\n"), + conn, + ) + .await; + } + } let reconciled = reconcile_materializations(&prepared, &results, job, conn, client).await; terminalize_running_relations(job, &reconciled, conn).await; @@ -653,12 +826,26 @@ pub(crate) async fn dbt_dep( None => GraphPublisher::Unversioned, }; let superseded = if let Some(warehouse) = prepared.warehouse.as_deref() { - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, prepared.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + &prepared, + &descriptor, + &inv, + // A deploy resolves the project by parsing it; nothing is built, so + // the pass takes the descriptor's own answer. + "parse", + &mut ctx, + job_id, + w_id, + &conn, + ) + .await?; let published = persist_ingest( db, w_id, @@ -682,6 +869,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -866,6 +1054,9 @@ impl GraphRefresh { if selection_is_overridden(descriptor, args)? { self.per_run_models = true; } + if full_refresh_is_overridden(descriptor, args)? { + self.per_run_models = true; + } Ok(()) } } @@ -894,6 +1085,17 @@ pub struct PreparedProject { /// The descriptor's `profile.target`, passed as `--target` so it applies to /// a project-owned `profiles.yml` as well as a rendered one. pub target: Option, + /// The target dbt actually runs, which is the above only when the descriptor + /// names one: otherwise it is the workspace warehouse's, or the project's own + /// `profiles.yml` default. Half of an environment's identity, since a + /// `target.name` macro decides where a model is built. + pub effective_target: Option, + /// Whether the profile templates where its relations go — a project-owned + /// `profiles.yml`, a `dbt_profile` resource's block, or `profile.schema`, + /// all of which reach dbt as written. Two renderings then share one + /// `relation_root` and an environment cannot be told apart, so such a + /// project neither publishes state nor defers to any. + pub templated_location: bool, /// The profile target's database. Nodes that override it qualify their /// `dbt://` schema segment so two databases cannot collapse onto one node. pub default_database: Option, @@ -919,19 +1121,76 @@ pub struct PreparedProject { /// Written nsjail profile for this job, when the worker sandboxes jobs. /// `None` means the phases run unsandboxed, exactly as before. pub sandbox_config: Option, - /// One-way digest of the rendered profile — the resolved connection, not - /// just the names it exposes. A resource repointed from one warehouse to - /// another that happens to use the same database and schema names is - /// invisible to `relation_root`, and a retry would then execute the saved - /// failures against a warehouse where the successful nodes do not exist. + /// One-way digest of the rendered profile, credentials masked: the resolved + /// connection, not just the names it exposes. A resource repointed from one + /// warehouse to another that happens to use the same database and schema + /// names is invisible to `relation_root`, and a retry would then execute the + /// saved failures against a warehouse where the successful nodes do not exist. pub profile_digest: String, + /// The job's client, which `refresh_profile` re-resolves the warehouse with. + client: AuthedClient, } impl PreparedProject { + /// Re-resolve the warehouse and rewrite `profiles.yml` just before a dbt + /// process that logs in. What preparation wrote can have expired by then: a + /// Snowflake OAuth token lasts ten minutes, and the build follows `dbt deps` + /// and a parse, the `after_all` tests follow the build, a node retry follows + /// its backoff. + pub(crate) async fn refresh_profile( + &self, + descriptor: &DbtDescriptor, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + ) -> error::Result<()> { + // A project-owned `profiles.yml` is rendered by dbt itself, from an + // environment resolved once. + if descriptor.profile.profiles_yml.is_some() { + return Ok(()); + } + let fresh = match write_profiles( + descriptor, + &self.project_dir, + &self.project_dir.to_string_lossy(), + &self.client, + &self.template_env(), + ) + .await + { + Ok(fresh) => fresh, + // The profile on disk is still whole: a credential that does not + // expire connects with it exactly as before. + Err(e) => { + append_logs( + job_id, + w_id, + format!( + "\nCould not re-resolve the warehouse, so this dbt process uses the \ + credentials resolved earlier in the job: {e}\n" + ), + conn, + ) + .await; + return Ok(()); + } + }; + // Credentials are masked out of the digest, so a mismatch is the warehouse + // itself moving mid-run, and this process would build somewhere else. + if fresh.digest != self.profile_digest { + return Err(Error::BadRequest(format!( + "the `{}` warehouse was repointed while this run was in progress; run the \ + script again", + self.warehouse.as_deref().unwrap_or(DBT_DEFAULT_WAREHOUSE) + ))); + } + Ok(()) + } + /// Where this run's relations live: the resolved schema and database. Drift /// here since the deploy means the stored graph names relations that no /// longer exist. - fn relation_root(&self) -> String { + pub(crate) fn relation_root(&self) -> String { format!( "{}|{}", self.default_schema.as_deref().unwrap_or(""), @@ -1063,8 +1322,8 @@ pub(crate) async fn prepare_project( .chain(invocation_env.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect(); - let (profiles_dir, warehouse, adapter, default_database, default_schema, profile_digest) = - write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let profile = write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let adapter = profile.adapter.clone(); // The lockfile's version, when it pinned one for this same engine — a // descriptor edited to another engine invalidates the pin. let pinned_version = locks @@ -1185,18 +1444,21 @@ pub(crate) async fn prepare_project( h.finish() }, sandbox_config, - profile_digest, + profile_digest: profile.digest, + client: client.clone(), project_dir, - profiles_dir, + profiles_dir: profile.dir, engine, graph_refresh, - warehouse, + warehouse: profile.warehouse, target: descriptor.profile.target.clone(), + effective_target: profile.target, + templated_location: profile.templated_location, descriptor_content: descriptor_content.to_string(), descriptor_env, - default_database, - default_schema, + default_database: profile.database, + default_schema: profile.schema, script_path: script_path.to_string(), env, }; @@ -1511,6 +1773,24 @@ async fn strip_git_remote(dir: &Path) -> std::io::Result<()> { tokio::fs::write(&config, out).await } +/// What resolving the run's connection settled, beyond the file itself. +struct ResolvedProfile { + dir: PathBuf, + /// The workspace warehouse's NAME, when this project belongs to one. + warehouse: Option, + adapter: DbtAdapter, + database: Option, + schema: Option, + /// The target dbt actually runs, which is not always the descriptor's: it + /// falls back to the workspace warehouse's, and to the project's own + /// `profiles.yml` default. Resolved because it is half of an environment's + /// identity and a `target.name` macro can move every relation. + target: Option, + /// Whether a project-owned `profiles.yml` templates where its relations go. + templated_location: bool, + digest: String, +} + /// Write `profiles.yml`, either rendered from a Windmill resource or taken from /// the project itself. Both paths are supported (decision 8): the workspace /// warehouse is the ergonomic one, the project's own file is what makes an @@ -1521,14 +1801,7 @@ async fn write_profiles( job_dir: &str, client: &AuthedClient, template_env: &HashMap, -) -> error::Result<( - PathBuf, - Option, - DbtAdapter, - Option, - Option, - String, -)> { +) -> error::Result { // The workspace's warehouse, always: a descriptor names one by NAME or takes // `main`, and cannot name a resource at all. The NAME is what asset identity // keys on, so every project on one warehouse shares its nodes while the @@ -1609,14 +1882,16 @@ async fn write_profiles( } None => None, }; - return Ok(( + return Ok(ResolvedProfile { dir, - identity, + warehouse: identity, adapter, - target.database, - target.schema, - profile_digest, - )); + database: target.database, + schema: target.schema, + target: Some(target.name), + templated_location: target.templated_location, + digest: profile_digest, + }); } use windmill_common::workspaces::DBT_PROFILE_RESOURCE_TYPE; @@ -1666,9 +1941,6 @@ async fn write_profiles( .or(workspace_target.as_deref()) .unwrap_or("default"); let dir = PathBuf::from(job_dir).join("dbt_profiles"); - tokio::fs::create_dir_all(&dir) - .await - .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; let rendered = if is_dbt_profile { let block = value.as_object().ok_or_else(|| { Error::BadRequest( @@ -1689,6 +1961,7 @@ async fn write_profiles( } else { render_profile( &adapter, + descriptor.engine(), &value, &profile_name, target, @@ -1697,6 +1970,10 @@ async fn write_profiles( &dir, )? }; + // After the render, so a render that fails leaves the previous profile whole. + fresh_dir(&dir) + .await + .map_err(|e| Error::internal_err(format!("creating the profiles dir: {e}")))?; write_file(dir.to_str().unwrap(), "profiles.yml", &rendered.yaml)?; if let Some(pem) = rendered.root_certificate_pem.as_deref() { write_file( @@ -1706,19 +1983,41 @@ async fn write_profiles( )?; } let profile_digest = profile_identity_digest( - &rendered.yaml, + &rendered.identity, &dir, rendered.root_certificate_pem.as_deref(), &client.token, ); - Ok(( + Ok(ResolvedProfile { dir, - Some(warehouse.to_string()), + warehouse: Some(warehouse.to_string()), adapter, - rendered.database, - rendered.schema, - profile_digest, - )) + // A `dbt_profile` resource is one block of the user's own + // `profiles.yml`, copied through unchanged, and `profile.schema` is + // written as given — so either can be a template dbt renders and this + // runtime does not, exactly as a project-owned file can. + templated_location: [rendered.database.as_deref(), rendered.schema.as_deref()] + .iter() + .any(|v| v.is_some_and(is_jinja)), + database: rendered.database, + schema: rendered.schema, + target: Some(target.to_string()), + digest: profile_digest, + }) +} + +/// An empty directory at `dir`, whatever was there. `refresh_profile` writes into +/// the job directory after project code has run in a jail that can write it, and a +/// symlink left at `dir` or inside it would carry the worker's write out of the +/// sandbox. An entry that is not a real directory is unlinked, never followed. +async fn fresh_dir(dir: &Path) -> std::io::Result<()> { + match tokio::fs::symlink_metadata(dir).await { + Ok(m) if m.is_dir() => tokio::fs::remove_dir_all(dir).await?, + Ok(_) => tokio::fs::remove_file(dir).await?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + tokio::fs::create_dir_all(dir).await } /// Where a workspace warehouse name points: its resource path and, if the @@ -1740,7 +2039,9 @@ async fn resolve_warehouse( .map_err(|e| Error::BadRequest(format!("resolving the dbt warehouse `{warehouse}`: {e}"))) } -/// Identifies the connection a rendered profile describes, for run identity. +/// Identifies the connection a rendered profile describes, for run identity, +/// from the rendering whose credentials are already masked +/// (`RenderedProfile::identity`). /// /// Two things in the rendered text belong to the ATTEMPT rather than the /// connection, and hashing either as-is makes a retry reject its own @@ -1859,13 +2160,45 @@ async fn adapter_from_profiles_yml( // identically to one on a workspace warehouse, which is what lets the two // meet on the same node when they are on the same relation. let (database_key, schema_key) = adapter.target_identity_keys(); - let read = |k: &str| { + let raw = |k: &str| { out.get(k) .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - .filter(|v| !v.is_empty() && !v.contains("{{")) + .filter(|v| !v.is_empty()) }; - Ok(ProfileTarget { adapter, database: read(database_key), schema: read(schema_key) }) + let read = |k: &str| raw(k).filter(|v| !v.contains("{{")).map(|v| v.to_string()); + Ok(ProfileTarget { + adapter, + database: read(database_key), + schema: read(schema_key), + // A TEMPLATED location is one dbt renders and this runtime does not, so + // two renderings of this file resolve to one `relation_root` and would + // share one environment — `{{ }}` because `read` drops it and it reads + // as absent, `{% %}` because the raw block is kept and reads the same + // for every rendering. Distinguished from plainly absent, which is the + // adapter's default and does not move. + templated_location: [database_key, schema_key] + .iter() + .any(|k| raw(k).is_some_and(is_jinja)), + // The output actually chosen, which for a templated `target:` is the sole + // one rather than the template text no output answers to. + name: match ( + templated_target, + outputs.as_mapping().and_then(|m| m.keys().next()), + ) { + (true, Some(only)) => only.as_str().unwrap_or(target).to_string(), + _ => target.to_string(), + }, + }) +} + +/// Whether dbt would RENDER this value rather than take it literally. +/// +/// Both delimiters, because dbt renders a profile through Jinja: `{{ … }}` +/// substitutes and `{% … %}` branches, and a schema spelled +/// `{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}` moves +/// every relation exactly as an `env_var()` does. +fn is_jinja(v: &str) -> bool { + v.contains("{{") || v.contains("{%") } /// What a project-owned `profiles.yml` target says, for the two things Windmill @@ -1876,6 +2209,14 @@ struct ProfileTarget { adapter: DbtAdapter, database: Option, schema: Option, + /// The output this resolved to, by name. + name: String, + /// Whether its database or schema is a template rather than a literal. The + /// fields above cannot say: a `{{ }}` value is dropped and reads as absent, + /// a `{% %}` block is kept and reads the same for every rendering. So this + /// is what separates "the adapter's default, which does not move" from + /// "wherever this run's environment renders it to". + templated_location: bool, } lazy_static::lazy_static! { @@ -2006,7 +2347,11 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command { .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("GIT_PATH", GIT_PATH.as_str()); + .env("GIT_PATH", GIT_PATH.as_str()) + // dbt reports anonymous usage to dbt Labs from every invocation unless told + // not to, and a project's `flags:` block cannot override the variable. Set + // before the project's environment so a descriptor can still opt back in. + .env("DBT_SEND_ANONYMOUS_USAGE_STATS", "false"); // Both environments belong to the child. Under a sandbox they reach it through // the jail profile instead: set here, they would reach the dynamic loader that // execs nsjail itself, so an `LD_PRELOAD` from the project would run as the @@ -2191,6 +2536,29 @@ async fn retry_failed_nodes( } } +/// The flags that point a deferring invocation at its state directory. +/// +/// `--state` is where a deferred `ref()` resolves through — except on a `retry`, +/// which reads the run it RESUMES from that same flag: handed the deferral's +/// directory, dbt resumes the successful run stored there and rebuilds nothing. +/// dbt-core 1.x has `--defer-state` for exactly this split; the Rust engines do +/// not, and a run that defers is refused a retry there rather than rebuilt with +/// its refs resolving into the schema it writes into (`handle_dbt_job`), which +/// is why the last arm never fires in practice. +/// +/// The directory is relative because dbt records the invocation's flags into +/// `run_results.json`: an absolute path would name the job directory of the run +/// being resumed, gone by the time anything reads it back. +fn defer_flags(command: &str, engine: DbtEngine) -> &'static [&'static str] { + match command { + // `--defer` itself is restored with the rest of the resumed + // invocation's arguments and cannot be set from here. + "retry" if engine.has_defer_state_flag() => &["--defer-state", STATE_DIR], + "retry" => &[], + _ => &["--defer", "--state", STATE_DIR], + } +} + #[allow(clippy::too_many_arguments)] async fn run_dbt( p: &PreparedProject, @@ -2202,6 +2570,8 @@ async fn run_dbt( ctx: &mut JobCtx<'_>, with_selection: bool, ) -> error::Result<()> { + p.refresh_profile(descriptor, &job.id, &job.workspace_id, conn) + .await?; let mut cmd = dbt_command(p, &[command]); // The console stays human-readable and goes straight to the job log; the // machine-readable copy goes to a file the progress reporter tails, so @@ -2212,6 +2582,10 @@ async fn run_dbt( .args(["--log-format-file", "json"]) .args(["--log-level-file", p.engine.engine.progress_log_level()]); + if inv.deferral.is_some() { + cmd.args(defer_flags(command, p.engine.engine)); + } + if with_selection && command != "retry" { add_selection(&mut cmd, descriptor, inv)?; } @@ -2232,8 +2606,7 @@ async fn run_dbt( if let Some(t) = descriptor.threads { cmd.args(["--threads", &t.to_string()]); } - let full_refresh = arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh); - if full_refresh && command != "test" { + if full_refresh(descriptor, inv, command)? { cmd.arg("--full-refresh"); } } @@ -2862,7 +3235,11 @@ async fn run_show( `@`) or wildcard (`*`), resolves to a set: run `build` with it instead" ))); } + p.refresh_profile(descriptor, job_id, w_id, conn).await?; let mut cmd = dbt_command(p, &["show"]); + if inv.deferral.is_some() { + cmd.args(defer_flags("show", p.engine.engine)); + } add_vars(&mut cmd, descriptor, inv)?; // Intersected with `resource_type:model`, because `show` is only read-only // for models: dbt dispatches a selected SEED through its seed runner and @@ -2882,7 +3259,8 @@ async fn run_show( conn, SHOW_MAX_OUTPUT_BYTES, ) - .await?; + .await? + .stdout; // dbt frames the rows as `{"node": …, "show": [ … ]}`, pretty-printed, with a // banner before and a deprecation summary after — so neither "the line starting // with `{`" nor "first `{` to the end" parses. A streaming deserializer stops at @@ -2956,6 +3334,7 @@ fn build_result( totals, nodes, invocation_args: inv.raw_args.clone(), + deferred_to: inv.deferral.as_ref().map(|d| d.published_by), } } @@ -3064,7 +3443,7 @@ async fn run_parse_only( // manifest and the selection while the warehouse only keys them — so a project // with no warehouse identity still reports what dbt found. The placeholder // reaches no row: the guard below returns before anything is written. - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, p.warehouse.as_deref().unwrap_or("unkeyed"), p.default_database.as_deref(), @@ -3084,6 +3463,20 @@ async fn run_parse_only( else { return Ok(to_raw_value(&result)); }; + // AFTER the guard: the pass is a second `dbt compile` and a parquet decode, + // and a parse that stores nothing has nowhere to put what it would produce. + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + "parse", + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; match conn { Connection::Sql(db) => match job.runnable_id.map(|h| h.0) { Some(script_hash) => { @@ -3143,11 +3536,70 @@ async fn run_parse_only( Ok(to_raw_value(&result)) } +/// Fold this project's column lineage into the graph about to be stored, when +/// the descriptor asked for it. +/// +/// One helper for all three ingests — deploy, editor parse, per-run refresh — +/// because a graph that carries column lineage in one provenance and not another +/// reads as the lineage having disappeared. +async fn attach_column_index( + ingested: &mut windmill_common::dbt_manifest::IngestedManifest, + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result<()> { + // The nodes this graph kept, so the pass reads only rows it could store: the + // index describes the whole project, this graph one selection of it. + let kept: std::collections::HashSet<&str> = ingested + .nodes + .iter() + .map(|n| n.unique_id.as_str()) + .collect(); + let index = + crate::dbt_column_index::collect( + p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept, + ) + .await?; + drop(kept); + let Some(index) = index else { + return Ok(()); + }; + let found = index.edges.len(); + ingested.attach_column_index(index); + let kept = ingested.column_edges.len(); + let typed: usize = ingested + .nodes + .iter() + .filter(|n| n.column_schema.is_some()) + .count(); + // Counted here rather than at the pass: the index describes the whole + // project and this graph describes one selection of it, so `found` is what + // dbt produced and `kept` is what the graph can draw. + let dropped = match found.saturating_sub(kept) { + 0 => String::new(), + n => format!(" ({n} outside this graph or past the cap)"), + }; + append_logs( + job_id, + w_id, + format!("\nIngested {kept} column lineage edges{dropped} and typed {typed} nodes\n"), + conn, + ) + .await; + Ok(()) +} + /// Refresh the stored graph from the manifest this run produced. async fn ingest_from_run( p: &PreparedProject, descriptor: &DbtDescriptor, inv: &Invocation, + command: &str, ctx: &mut JobCtx<'_>, job: &MiniPulledJob, conn: &Connection, @@ -3164,12 +3616,24 @@ async fn ingest_from_run( // filter this run's manifest by a different node set than it built. let selected = resolve_selection(p, descriptor, inv, ctx, &job.id, &job.workspace_id, conn).await?; - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, p.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + command, + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; // Only a run whose models are its own snapshots per run. A static // descriptor at a moved profile re-ingests the VERSION's graph, since the // move outlives the run; one that neither drifted nor overrode anything @@ -3177,7 +3641,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3190,6 +3654,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3242,7 +3713,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3333,9 +3804,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3349,6 +3821,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and @@ -3411,6 +3933,12 @@ async fn resolve_selection( return Ok(None); } let mut cmd = dbt_command(p, &["ls"]); + // The same state the build resolves through, or a `result:` selector — which + // reads `run_results.json` out of it, and which `select` passes to dbt + // verbatim — fails here, before the build that would have honoured it. + if inv.deferral.is_some() { + cmd.args(defer_flags("ls", p.engine.engine)); + } // A project whose models call `var()` without a default fails to parse // without these, so the selection resolver needs them exactly as the run // does. Placeholders that only a run can fill are dropped rather than @@ -3424,11 +3952,15 @@ async fn resolve_selection( } cmd.args(["--output", "json", "--quiet"]); add_selection(&mut cmd, descriptor, inv)?; + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; // Captured directly, not through `handle_child`: its `pipe_stdout` path goes // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection // would resolve to the empty set and the ingest would wipe the script's assets // while dbt went on building the descriptor's models. - let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES).await?; + let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES) + .await? + .stdout; let mut set = std::collections::HashSet::new(); for line in stdout.lines() { let line = line.trim(); @@ -3441,15 +3973,37 @@ async fn resolve_selection( } } } - if set.is_empty() { - // A selection that matches nothing would be ingested as "this script - // owns no relations", wiping its graph and cascade edges — the same - // outcome a failed capture produces, and indistinguishable from it. - // Refuse rather than silently un-wire the script. + // Empty is a real answer from a `state:` or `result:` method and from nothing + // else: `state:modified+` matches nothing exactly when nothing changed since + // the published state, and a run with no work to do is a successful one. Any + // other selection matching nothing is a selector that names nothing — a + // misspelled model, say — which must not pass as a build that did its job. + // Exempting by ORIGIN rather than by method would let every such typo through. + // + // What makes the exemption safe is that the empty set is never ingested as + // ownership, and that now holds through `check_state_selectors`: a `state:` + // or `result:` method survives it only from a run's OWN selection, which + // makes `add_caller_args` set `per_run_models`, which makes + // `publishes_ownership()` false, so the run stores a snapshot of its own. + // Relax the descriptor arm there and a descriptor-narrowed `state:modified+` + // reaches here on an unchanged project and wipes the graph the `else` below + // guards, with nothing failing. + if set.is_empty() && !selection_names(&select, &exclude, &["state", "result"]) { return Err(Error::ExecutionErr( - "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection rather \ - than deploying a script that owns nothing" - .to_string(), + if selection_is_overridden(descriptor, &inv.args)? { + "this run's `select`/`exclude` matched no dbt nodes, so it would build nothing; \ + check the selector. Only a `state:` or `result:` selector may match nothing, \ + its empty answer being a real one" + .to_string() + } else { + // The descriptor's is also ingested as "this script owns no + // relations", wiping its graph and cascade edges — the same + // outcome a failed capture produces, and indistinguishable from + // it. Refuse rather than silently un-wire the script. + "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection \ + rather than deploying a script that owns nothing" + .to_string() + }, )); } Ok(Some(set)) @@ -3459,6 +4013,34 @@ async fn resolve_selection( /// what is kept is the TAIL, because dbt prints its error summary last. const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; +/// What a captured invocation produced. `stderr` is where dbt writes its +/// diagnostics — the errors and warnings block — so a caller that has to explain +/// a SUCCESSFUL run needs it as much as a failing one does. +pub(crate) struct Captured { + pub stdout: String, + pub stderr: String, + /// Whether the child exited zero. Separate from the `Result` on purpose: an + /// `Err` from `run_captured` is the JOB's — a cancellation or its deadline — + /// so a caller that tolerates a failed command must still propagate one. + pub success: bool, + /// Whether the output ceiling cut the child short. Only ever true under + /// [`Overflow::Truncate`]. + pub truncated: bool, +} + +/// What an over-long stdout means to the caller. +/// +/// The ceiling belongs to the PASS, not to the job: a caller that only annotates +/// a job wants to keep what it read and carry on, while one whose whole result +/// is that output has nothing to return without it. +#[derive(PartialEq, Eq, Clone, Copy)] +pub(crate) enum Overflow { + /// Fail the job. For a command whose output IS the answer. + Fail, + /// Stop reading, kill the child, and report `truncated`. + Truncate, +} + /// Run a command for its stdout under the job's cancellation and timeout. /// The same poller `handle_child` uses drives them, so a cancel or a deadline /// drops the wait future — which owns the child, and `kill_on_drop` then @@ -3471,7 +4053,7 @@ const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; /// never holds more than it, so it has to be enforced while reading. Both pipes /// are drained concurrently because a child that fills the one nobody reads /// blocks forever. -async fn run_capturing( +pub(crate) async fn run_captured( mut cmd: Command, name: &str, ctx: &mut JobCtx<'_>, @@ -3479,7 +4061,8 @@ async fn run_capturing( w_id: &str, conn: &Connection, max_stdout_bytes: usize, -) -> error::Result { + on_overflow: Overflow, +) -> error::Result { use tokio::io::AsyncReadExt; let mut child = cmd @@ -3514,6 +4097,7 @@ async fn run_capturing( let mut out_buf = vec![0u8; 16 * 1024]; let mut err_buf = vec![0u8; 16 * 1024]; let (mut out_open, mut err_open) = (true, true); + let mut truncated = false; while out_open || err_open { tokio::select! { r = stdout_pipe.read(&mut out_buf[..]), if out_open => match r { @@ -3521,14 +4105,19 @@ async fn run_capturing( Ok(n) => { if stdout.len() + n > max_stdout_bytes { // Killed here rather than left to `kill_on_drop` - // so the child is gone before the error unwinds, - // not merely once this future is dropped. + // so the child is gone before this returns, not + // merely once the future is dropped. let _ = child.kill().await; - return Err(Error::ExecutionErr(format!( - "{name} produced more than {} MB of output. Narrow the \ - selection, or query the relation from a SQL script.", - max_stdout_bytes / 1024 / 1024 - ))); + if on_overflow == Overflow::Fail { + return Err(Error::ExecutionErr(format!( + "{name} produced more than {} MB of output. Narrow the \ + selection, or query the relation from a SQL script.", + max_stdout_bytes / 1024 / 1024 + ))); + } + truncated = true; + out_open = false; + continue; } stdout.extend_from_slice(&out_buf[..n]); } @@ -3551,7 +4140,7 @@ async fn run_capturing( .wait() .await .map_err(|e| Error::internal_err(format!("{name} failed: {e}")))?; - Ok((status, stdout, stderr)) + Ok((status, stdout, stderr, truncated)) }, ctx.worker_name, w_id, @@ -3561,14 +4150,46 @@ async fn run_capturing( })), ) .await?; - let (status, stdout, stderr) = out; - if !status.success() { + let (status, stdout, stderr, truncated) = out; + Ok(Captured { + stdout: String::from_utf8_lossy(&stdout).to_string(), + stderr: String::from_utf8_lossy(&stderr).to_string(), + // A killed child reports failure; under `Truncate` that is the ceiling's + // doing, not the project's, and the caller reads `truncated` to tell. + success: status.success(), + truncated, + }) +} + +/// `run_captured`, with a non-zero exit folded into the error — what a caller +/// that needs the command to have WORKED wants. +pub(crate) async fn run_capturing( + cmd: Command, + name: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + max_stdout_bytes: usize, +) -> error::Result { + let captured = run_captured( + cmd, + name, + ctx, + job_id, + w_id, + conn, + max_stdout_bytes, + Overflow::Fail, + ) + .await?; + if !captured.success { return Err(Error::ExecutionErr(format!( "{name} failed: {}", - String::from_utf8_lossy(&stderr) + captured.stderr ))); } - Ok(String::from_utf8_lossy(&stdout).to_string()) + Ok(captured) } /// Run a preparation command through the same child handler the build uses, so @@ -3761,7 +4382,7 @@ async fn save_run_state( if let Connection::Sql(db) = conn { { // Only while a live dbt version stays at this path — the test - // `clear_dbt_run_state_if_path_retired` retires state by, plus the + // `clear_dbt_script_state_if_path_retired` retires state by, plus the // language, since a rename leaves the old path archived rather than // deleted and a path can come back as another language. A job already // running finishes after those move or clear the row: writing then @@ -3923,6 +4544,12 @@ pub struct Invocation { /// what it pointed at must not. pub raw_args: HashMap>, pub envs: HashMap, + /// The stored dbt state this invocation resolves an unbuilt `ref()` through, + /// materialised into the job directory. Carried here rather than passed to + /// each phase: the model phase, the `after_all` tests and every in-job node + /// retry must all resolve a `ref()` the same way, or the tests assert against + /// relations the models never read. + pub deferral: Option, /// A run must fail on a `{{ }}` placeholder it cannot fill; a deploy, which /// has no arguments at all, tolerates them. Declared rather than inferred /// from the argument count: a run submitted with `{}` is still a run, and @@ -4075,11 +4702,12 @@ async fn restore_from_db( if !has_retryable_node(&row.run_results) { return Err(nothing_to_retry()); } - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - tokio::fs::write(target.join("run_results.json"), &row.run_results) - .await - .map_err(|e| Error::internal_err(format!("restoring run_results.json: {e}")))?; + write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&row.run_results), + StateManifest::None, + ) + .await?; // No manifest came with the row, so one has to be re-derived — but not here: // these arguments are as SUBMITTED, and a `$var:` in them shapes the graph // only once resolved. The caller resolves, then parses. @@ -4311,20 +4939,16 @@ async fn restore_run_state( return Err(different_project()); } let saved_args_digest = saved_args_digest.map(str::to_string); - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - // From the bytes already read, not by copying the file again: a burst of saves - // can prune this generation mid-restore, and a `dbt retry` whose + // The results go from the bytes already read, not by copying the file again: a + // burst of saves can prune this generation mid-restore, and a `dbt retry` whose // `run_results.json` went missing rebuilds nothing and reports success. The // manifest has no such copy, so a failure there falls back to a `dbt parse`. - tokio::fs::write(target.join("run_results.json"), &saved_results) - .await - .map_err(|e| { - Error::internal_err(format!("could not restore the previous run's results: {e}")) - })?; - let needs_parse = tokio::fs::copy(snapshot.join("manifest.json"), target.join("manifest.json")) - .await - .is_err(); + let needs_parse = !write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&saved_results), + StateManifest::CopyOf(snapshot.join("manifest.json")), + ) + .await?; // The generation was chosen from a row read before the file work above. A run // finishing in that window publishes a newer one, and resuming the superseded // generation redoes nodes it has already rebuilt — appending to an incremental @@ -4518,7 +5142,11 @@ fn has_retryable_node(run_results: &str) -> bool { } /// Append `--vars` if the descriptor (or the run) declares any. -fn add_vars(cmd: &mut Command, descriptor: &DbtDescriptor, inv: &Invocation) -> error::Result<()> { +pub(crate) fn add_vars( + cmd: &mut Command, + descriptor: &DbtDescriptor, + inv: &Invocation, +) -> error::Result<()> { let vars = resolved_vars(descriptor, &inv.args, inv.strict)?; if !vars.is_empty() { cmd.args(["--vars", &serde_json::to_string(&vars).unwrap_or_default()]); @@ -4713,10 +5341,26 @@ fn add_selection( descriptor: &DbtDescriptor, inv: &Invocation, ) -> error::Result<()> { - for s in effective_select(descriptor, inv)? { + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; + // The seam itself, which the DEPLOY reaches without going through a run: it + // resolves the descriptor's selection to decide what the script owns, and + // never computes a `defer`. A run has been checked earlier, where the message + // can still come before the state fetch. + check_state_selectors( + &select, + &exclude, + if inv.deferral.is_some() { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(descriptor, &inv.args)?, + )?; + for s in select { cmd.args(["--select", &s]); } - for s in effective_exclude(descriptor, inv)? { + for s in exclude { cmd.args(["--exclude", &s]); } if let Some(sel) = effective_selector(descriptor, inv)? { @@ -4725,6 +5369,115 @@ fn add_selection( Ok(()) } +/// The method a selection token names, with the graph operators that can +/// surround a node stripped (`@model`, `+model`, `2+model`, `model+`). +fn selector_method(token: &str) -> Option<&str> { + token + .trim_start_matches('@') + .trim_start_matches(|c: char| c.is_ascii_digit()) + .trim_start_matches('+') + .split_once(':') + .map(|(method, _)| method) +} + +/// Every method a selection names. Each entry is a union of whitespace-separated +/// tokens, and each of those an intersection of comma-separated ones. +fn selection_methods<'a>(entries: &'a [String]) -> impl Iterator { + entries + .iter() + .flat_map(|entry| entry.split([' ', '\t', ','])) + .filter_map(selector_method) +} + +/// Whether the run being checked has the state directory a `state:` or `result:` +/// method reads, or could be given one. +#[derive(Clone, Copy)] +enum StateAccess<'a> { + /// Deferring, so the directory is there. + Given, + /// Not deferring, and `defer` is what would hand it one. + OnRequest, + /// This command resolves a selection without ever deferring, so no setting + /// gives it a state and "turn `defer` on" would be advice that leads nowhere. + Never(&'a str), +} + +/// Whether a selection names any of these methods. +fn selection_names(select: &[String], exclude: &[String], methods: &[&str]) -> bool { + selection_methods(select) + .chain(selection_methods(exclude)) + .any(|method| methods.contains(&method)) +} + +/// Refuse a selection dbt cannot resolve, before it silently resolves to the +/// wrong thing. +/// +/// `state:` and `result:` compare against the artifacts in `--state`, which only +/// a deferring run is given. The engines do not agree on what happens without +/// one: dbt-core 1.x raises, but dbt-sa-cli and fusion read a missing state as an +/// EMPTY one and exit 0, so `state:modified` builds nothing and `state:new` +/// builds the whole project, each as a run that reports success. +/// +/// From the DESCRIPTOR they are refused whether or not the run defers, because +/// that selection also decides which nodes the script owns, and "whatever changed +/// last" is not an ownership answer — the deploy resolves it with no state at all. +/// They describe one run, so they belong in a run's own `select`. +/// +/// `source_status:` compares `sources.json`, which `dbt source freshness` writes +/// and no run publishes here, so it has nothing to compare against under any +/// setting. +/// +/// Only what `select` and `exclude` spell directly: a method reached through a +/// `selectors.yml` definition is named nowhere the worker can read, and dbt's +/// own behaviour is what stands there. +fn check_state_selectors( + select: &[String], + exclude: &[String], + access: StateAccess<'_>, + from_descriptor: bool, +) -> error::Result<()> { + for method in selection_methods(select).chain(selection_methods(exclude)) { + match method { + "source_status" => { + return Err(Error::BadRequest( + "a `source_status:` selector compares the source freshness recorded in \ + `sources.json`, which `dbt source freshness` writes and no run stores \ + here, so there is nothing for it to compare against. Drop the selector" + .to_string(), + )) + } + "state" | "result" if from_descriptor => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector describes what ONE run builds, but the descriptor's \ + selection also decides which nodes this script owns, which a deploy \ + resolves with no state to compare against. Move it to the `select` of a \ + run with `defer` on" + ))) + } + "state" | "result" => match access { + StateAccess::Given => {} + StateAccess::OnRequest => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and only a run with `defer` on is given \ + that state. Turn `defer` on, or drop the selector" + ))) + } + StateAccess::Never(command) => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and `{command}` resolves its selection \ + without building and never defers, so no setting hands it that state. \ + Drop the selector" + ))) + } + }, + _ => {} + } + } + Ok(()) +} + /// Whether this invocation chose its own `select`/`exclude`. /// /// DIFFERENT from the descriptor's, not merely present: `parse_dbt_sig` gives @@ -4748,6 +5501,41 @@ fn selection_is_overridden( Ok(differs("select", &descriptor.select)? || differs("exclude", &descriptor.exclude)?) } +/// Whether this invocation rebuilds incremental models from scratch: the run +/// form's answer when it gave one, else the descriptor's — and never for a +/// `test`, which builds nothing whatever the form said. +/// +/// Shared with the column-lineage pass rather than recomputed there, because +/// `is_incremental()` branches on it: the same model compiles to different SQL — +/// a `{{ this }}` self-join, and any `ref()` inside the incremental branch — so a +/// pass that guessed would describe a build that never ran. +/// +/// `test` returns false because `dbt test` rejects `--full-refresh` outright. +/// It never arrives as a caller's `dbt_command` — the allowlist has no such +/// value — so reading only that allowlist suggests this branch is dead. It is +/// not: `run_dbt` is invoked with `"test"` directly for the `after_all` test +/// phase, and an `after_all` project with `full_refresh: true` reaches here. +pub(crate) fn full_refresh( + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, +) -> error::Result { + if command == "test" { + return Ok(false); + } + Ok(arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh)) +} + +/// Whether this run answered `full_refresh` differently from the deployed +/// descriptor. Like a selection override it changes what the graph describes, +/// since an incremental branch can carry its own `ref()`. +fn full_refresh_is_overridden( + descriptor: &DbtDescriptor, + args: &HashMap>, +) -> error::Result { + Ok(arg_bool(args, "full_refresh")?.is_some_and(|v| v != descriptor.full_refresh)) +} + /// The descriptor's named selector, unless this run named its own selection. /// /// dbt resolves `--selector` INSTEAD of `--select`, so passing both makes the @@ -5007,20 +5795,47 @@ mod tests { // and without normalizing it the saved run is never recognized as its own. #[test] fn profile_identity_ignores_the_attempts_token() { - let yaml = |tok: &str| format!("host: \"wh\"\npassword: \"{tok}\"\n"); + let yaml = |v: &str| format!("host: \"{v}\"\nuser: \"u\"\n"); let dir = Path::new("/tmp/windmill/w/job-1/profiles"); assert_eq!( profile_identity_digest(&yaml("tok-first"), dir, None, "tok-first"), profile_identity_digest(&yaml("tok-retry"), dir, None, "tok-retry") ); - // A password that is NOT the job's token is the connection, and changing - // it must still read as a different warehouse. + // A value that is NOT the job's token is the connection, and changing it + // must still read as a different warehouse. assert_ne!( profile_identity_digest(&yaml("static-a"), dir, None, "tok-first"), profile_identity_digest(&yaml("static-b"), dir, None, "tok-retry") ); } + // Project code can leave a symlink where the worker later writes the profile: + // either the directory or the file in it. Neither may be followed. + #[cfg(unix)] + #[tokio::test] + async fn fresh_dir_never_follows_what_the_jail_left() { + let root = tempfile::tempdir().unwrap(); + let host = root.path().join("host"); + std::fs::create_dir(&host).unwrap(); + std::fs::write(host.join("profiles.yml"), "host file").unwrap(); + let dir = root.path().join("dbt_profiles"); + + std::os::unix::fs::symlink(&host, &dir).unwrap(); + fresh_dir(&dir).await.unwrap(); + assert!(!std::fs::symlink_metadata(&dir).unwrap().is_symlink()); + std::fs::write(dir.join("profiles.yml"), "rendered").unwrap(); + + fresh_dir(&dir).await.unwrap(); + std::os::unix::fs::symlink(host.join("profiles.yml"), dir.join("profiles.yml")).unwrap(); + fresh_dir(&dir).await.unwrap(); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + + assert_eq!( + std::fs::read_to_string(host.join("profiles.yml")).unwrap(), + "host file" + ); + } + // The jail profile is protobuf text format, and the project path and the // descriptor's environment land inside string literals. An unescaped quote or // newline closes the literal and lets the rest be read as further directives — @@ -5738,6 +6553,79 @@ mod tests { .unwrap(); assert!(untouched.publishes_ownership()); assert_eq!(untouched.snapshot_job(job), None); + + // `resolve_selection` lets a selection match nothing on exactly this + // predicate, because a run that scoped its own selection stores a + // snapshot instead of publishing ownership. Should the two ever drift + // apart, an empty caller selection would wipe the script's graph and + // cascade edges, which is the outcome that guard exists to prevent. + // One-directional: a `vars` override also withholds ownership without + // touching the selection, which is why this is an implication and not an + // equivalence. + for args in [ + arg("select", r#"["state:modified+"]"#), + arg("exclude", r#"["tag:nightly"]"#), + ] { + assert!(selection_is_overridden(&descriptor, &args).unwrap()); + let mut g = GraphRefresh::default(); + g.add_caller_args(&descriptor, &args).unwrap(); + assert!( + !g.publishes_ownership(), + "an overridden selection must not publish ownership" + ); + } + + // `full_refresh` decides whether `is_incremental()` is true, so an + // incremental model's self-join — and any `ref()` inside that branch — + // exists in one answer and not the other. A run that flips it describes + // a different graph, and gets its own. + let mut refreshed = GraphRefresh::default(); + refreshed + .add_caller_args(&descriptor, &arg("full_refresh", "true")) + .unwrap(); + assert!(refreshed.needed()); + assert_eq!(refreshed.snapshot_job(job), Some(job)); + + // The same echo rule: the form posts the descriptor's own value back on + // every run, and reading that as an override would make each one + // caller-scoped. + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let mut echoed_flag = GraphRefresh { profile_drift: true, ..Default::default() }; + echoed_flag + .add_caller_args(&always, &arg("full_refresh", "true")) + .unwrap(); + assert_eq!(echoed_flag.snapshot_job(job), None); + } + + /// The build and the analysis pass read this through one function, so they + /// cannot disagree about which SQL the run compiles — including for `test`, + /// which rebuilds nothing whatever the descriptor or the form said. + #[test] + fn full_refresh_is_one_answer_for_the_build_and_the_pass() { + let inv = |args: HashMap>| Invocation { + args, + raw_args: Default::default(), + envs: Default::default(), + strict: true, + deferral: None, + }; + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let never = DbtDescriptor::default(); + let on = HashMap::from([( + "full_refresh".to_string(), + RawValue::from_string("true".to_string()).unwrap(), + )]); + + assert!(full_refresh(&always, &inv(Default::default()), "build").unwrap()); + assert!(!full_refresh(&never, &inv(Default::default()), "build").unwrap()); + assert!( + full_refresh(&never, &inv(on), "build").unwrap(), + "the form's answer wins over the descriptor's" + ); + assert!( + !full_refresh(&always, &inv(Default::default()), "test").unwrap(), + "a test builds nothing, so neither the build nor the pass may pass the flag" + ); } // `dbt retry` restores the previous run's target/ from this directory, so two @@ -5766,6 +6654,128 @@ mod tests { ); } + // A profile whose location dbt renders cannot be told apart from another + // rendering of itself, so it neither publishes state nor defers. Both + // delimiters count: a conditional block moves a schema exactly as an + // `env_var()` substitution does. + #[test] + fn a_rendered_profile_location_is_recognised_by_either_delimiter() { + assert!(is_jinja("{{ env_var('DBT_SCHEMA') }}")); + assert!(is_jinja( + "{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}" + )); + assert!(!is_jinja("analytics")); + assert!(!is_jinja("")); + } + + // dbt-sa-cli and fusion exit 0 on a state selector with no state, so nothing + // downstream would report this: the graph operators have to be stripped for + // the method to be seen at all. + #[test] + fn a_state_selector_is_found_under_any_graph_operator() { + // A run's own selection, which is the only place these belong. + let refused = |sel: &str, access: StateAccess<'_>| { + check_state_selectors(&[sel.to_string()], &[], access, false).is_err() + }; + for sel in [ + "state:modified", + "state:modified+", + "+state:new", + "@state:modified", + "2+state:modified+3", + "tag:nightly,state:modified", + "stg_orders+ result:error+", + ] { + assert!( + refused(sel, StateAccess::OnRequest), + "{sel} should need `defer`" + ); + assert!( + !refused(sel, StateAccess::Given), + "{sel} should pass while deferring" + ); + // A parse resolves a selection without ever deferring, so it is + // refused where a run would have been told to turn `defer` on. + assert!( + refused(sel, StateAccess::Never("parse")), + "{sel} cannot parse" + ); + // The descriptor's selection also decides what the script owns, and + // the deploy resolves it with no state, so deferring cannot save it. + assert!( + check_state_selectors(&[sel.to_string()], &[], StateAccess::Given, true).is_err(), + "{sel} should never be a descriptor selection" + ); + } + // A node whose name merely starts with a method's letters is not one. + for sel in ["stg_orders+", "tag:nightly", "stateful_model+"] { + assert!( + !refused(sel, StateAccess::OnRequest), + "{sel} is not a state selector" + ); + } + // No run publishes `sources.json`, so deferring does not help. + for access in [ + StateAccess::Given, + StateAccess::OnRequest, + StateAccess::Never("parse"), + ] { + assert!(refused("source_status:fresher+", access)); + } + // `exclude` reaches dbt the same way `select` does. + assert!(check_state_selectors( + &[], + &["state:modified".to_string()], + StateAccess::OnRequest, + false + ) + .is_err()); + + // The same recognition decides which empty selections `resolve_selection` + // lets through. Only these two answer "nothing" meaningfully; a selector + // naming nothing must not pass as a build that did its work. + const STATE_BACKED: &[&str] = &["state", "result"]; + for sel in ["state:modified+", "result:error+", "tag:x,state:new"] { + assert!( + selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + for sel in [ + "mispelled_model", + "tag:nightly", + "stg_orders+", + "source_status:fresher+", + ] { + assert!( + !selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + } + + // The one flag choice that is silently wrong rather than loudly wrong: a + // `retry` handed `--state` resumes the SUCCESSFUL run stored there and + // rebuilds nothing, reporting a green retry of a failed run. + #[test] + fn a_retry_is_never_handed_the_deferral_as_its_state() { + assert_eq!( + defer_flags("build", DbtEngine::DbtCore1x), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("test", DbtEngine::Fusion), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("retry", DbtEngine::DbtCore1x), + ["--defer-state", crate::dbt_state::STATE_DIR] + ); + for engine in [DbtEngine::DbtCore2x, DbtEngine::Fusion] { + assert!(defer_flags("retry", engine).is_empty()); + } + } + #[test] fn events_without_a_relation_are_not_materializations() { // A test node has no relation of its own. diff --git a/backend/windmill-worker/src/dbt_profiles.rs b/backend/windmill-worker/src/dbt_profiles.rs index 3d96f84676..f81aea2ae5 100644 --- a/backend/windmill-worker/src/dbt_profiles.rs +++ b/backend/windmill-worker/src/dbt_profiles.rs @@ -7,12 +7,42 @@ use serde_json::Value; use windmill_common::error::{self, Error}; +use windmill_parser_yaml::dbt::DbtEngine; /// Written beside `profiles.yml`, and named absolutely in `sslrootcert`: dbt /// runs with the project as its working directory and hands the path to the /// driver unchanged. pub const ROOT_CERT_FILENAME: &str = "server-ca.pem"; +/// What a credential's value becomes in [`RenderedProfile::identity`]. +const MASKED_CREDENTIAL: &str = "$CREDENTIAL"; + +/// Whether a target key holds a credential rather than part of the address. By +/// name, because a `dbt_profile` block's keys are its adapter's own: `password`, +/// dbt-postgres's `pass`, `token`, `private_key_passphrase`, `client_secret`, +/// `aws_secret_access_key` and the `key_id` rotated with it, … An endpoint is +/// address even when its name says otherwise: BigQuery's `token_uri` is where the +/// token comes from. +fn is_credential_key(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + if key.ends_with("_uri") || key.ends_with("_url") || key.contains("endpoint") { + return false; + } + key == "pass" + || [ + "password", + "passphrase", + "secret", + "token", + "private_key", + "api_key", + "access_key", + "key_id", + ] + .iter() + .any(|c| key.contains(c)) +} + /// The per-adapter facts, so each adapter states them together and a new one /// cannot inherit another's by omission. `PG` is the base every arm spreads /// from: most adapters differ from Postgres only in their name and package. @@ -469,6 +499,10 @@ fn port_of(resource: &Value, default: i64) -> error::Result { #[derive(Debug)] pub struct RenderedProfile { pub yaml: String, + /// `yaml` with every credential's value masked: what run identity hashes. A + /// rotated token or password is still the same connection, and an OAuth token + /// rotates every few minutes, so hashing it would refuse nearly every retry. + pub identity: String, pub schema: Option, pub database: Option, /// A private CA the caller must write next to `profiles.yml`, under the @@ -483,8 +517,11 @@ pub struct RenderedProfile { /// from the descriptor when set, else from the resource, else from dbt's own /// per-adapter default — dbt errors out clearly when it ends up missing, which /// is a better failure than a Windmill-invented default. +#[allow(clippy::too_many_arguments)] pub fn render_profile( adapter: &DbtAdapter, + // The engines read some keys differently; see the Snowflake token below. + engine: DbtEngine, resource: &Value, profile_name: &str, target: &str, @@ -593,11 +630,18 @@ pub fn render_profile( out.push(("user".into(), quoted(&u))); } // Key-pair is Windmill's own snowflake resource shape; the - // `snowflake_oauth` type carries a token instead, which dbt only - // accepts alongside `authenticator: oauth` — without both, the - // profile renders with no credential at all and cannot connect. + // `snowflake_oauth` type carries an access token instead, which needs + // an `authenticator` saying so. dbt-core 1.x reads `oauth` + `token` + // as one. The Rust engines read `oauth` as the refresh-token flow and + // refuse a profile without client credentials, and send the same + // login for `jwt`, which dbt-snowflake has only from 1.9, while the + // 1.x engine still resolves 1.8. if let Some(t) = s(resource, "token").or_else(|| s(resource, "access_token")) { - out.push(("authenticator".into(), quoted("oauth"))); + let authenticator = match engine { + DbtEngine::DbtCore1x => "oauth", + DbtEngine::DbtCore2x | DbtEngine::Fusion => "jwt", + }; + out.push(("authenticator".into(), quoted(authenticator))); out.push(("token".into(), quoted(&t))); } else if let Some(k) = s(resource, "private_key") { out.push(("private_key".into(), quoted(&k))); @@ -612,10 +656,17 @@ pub fn render_profile( out.push((k.into(), quoted(&v))); } } - database = s(resource, "database"); - if let Some(d) = database.clone() { - out.push(("database".into(), quoted(&d))); - } + // dbt-snowflake requires one, and a resource from the OAuth connect flow + // starts without it: naming the field beats dbt's schema error. + let db = s(resource, "database").ok_or_else(|| { + Error::BadRequest( + "a Snowflake target needs a database; add `database` to the warehouse's \ + resource" + .to_string(), + ) + })?; + out.push(("database".into(), quoted(&db))); + database = Some(db); schema = schema.or_else(|| s(resource, "schema")); } KnownAdapter::Bigquery => { @@ -692,24 +743,43 @@ pub fn render_profile( // a newline in one opens a sibling key of the caller's choosing. let (qp, qt) = (yaml_scalar(profile_name), yaml_scalar(target)); let mut yaml = format!("{qp}:\n target: {qt}\n outputs:\n {qt}:\n"); + let mut identity = yaml.clone(); for (k, v) in &out { yaml.push_str(&format!(" {k}: {}\n", v.render())); + let shown = if is_credential_key(k) { + yaml_scalar(MASKED_CREDENTIAL) + } else { + v.render() + }; + identity.push_str(&format!(" {k}: {shown}\n")); } // The service-account document is a nested mapping, not a scalar. if adapter == KnownAdapter::Bigquery { yaml.push_str(" keyfile_json:\n"); + identity.push_str(" keyfile_json:\n"); let obj = resource .as_object() .ok_or_else(|| Error::BadRequest("bigquery resource is not an object".to_string()))?; for (k, v) in obj { if let Some(v) = v.as_str() { yaml.push_str(&format!(" {}: {}\n", yaml_scalar(k), yaml_scalar(v))); + let shown = if is_credential_key(k) { + MASKED_CREDENTIAL + } else { + v + }; + identity.push_str(&format!( + " {}: {}\n", + yaml_scalar(k), + yaml_scalar(shown) + )); } } } Ok(RenderedProfile { yaml, + identity, schema, database, root_certificate_pem: matches!(adapter, KnownAdapter::Postgres) @@ -746,6 +816,7 @@ pub fn render_dbt_profile( " \"type\": {}\n", yaml_scalar(adapter.dbt_type()) )); + let mut identity = yaml.clone(); for (k, v) in block { // A null is an optional field the resource form left unset, and dbt // validates several keys against a schema that rejects one. @@ -762,28 +833,33 @@ pub fn render_dbt_profile( if (k == schema_key && schema_override.is_some()) || (k == "threads" && threads.is_some()) { continue; } - emit_entry(&mut yaml, 6, k, v); + emit_entry(&mut yaml, 6, k, v, false); + emit_entry(&mut identity, 6, k, v, true); } + let mut tail = String::new(); if root_certificate_pem.is_some() { - yaml.push_str(&format!( + tail.push_str(&format!( " \"sslrootcert\": {}\n", yaml_scalar(&profiles_dir.join(ROOT_CERT_FILENAME).to_string_lossy()) )); } if let Some(sc) = schema_override { - yaml.push_str(&format!( + tail.push_str(&format!( " {}: {}\n", yaml_scalar(schema_key), yaml_scalar(sc) )); } if let Some(t) = threads { - yaml.push_str(&format!(" \"threads\": {t}\n")); + tail.push_str(&format!(" \"threads\": {t}\n")); } + yaml.push_str(&tail); + identity.push_str(&tail); let str_key = |k: &str| block.get(k).and_then(|v| v.as_str()).map(|v| v.to_string()); Ok(RenderedProfile { yaml, + identity, schema: schema_override .map(|x| x.to_string()) .or_else(|| str_key(schema_key)), @@ -794,16 +870,21 @@ pub fn render_dbt_profile( /// Emit one target key, nesting as deep as the value goes — an adapter's credential can be /// a mapping (bigquery's `keyfile_json`) or a list. Keys are quoted like values: one nothing -/// here enumerates is as free-form as a password. -fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value) { +/// here enumerates is as free-form as a password. `mask` writes credentials as +/// [`MASKED_CREDENTIAL`], at any depth, for [`RenderedProfile::identity`]. +fn emit_entry(out: &mut String, indent: usize, key: &str, v: &Value, mask: bool) { out.push_str(&format!("{}{}:", " ".repeat(indent), yaml_scalar(key))); - emit_value(out, indent, v); + emit_value(out, indent, v, mask, mask && is_credential_key(key)); } /// The value half, after `key:`. An empty collection is emitted INLINE: a block with no /// children reads back as `null`, so `extensions: []` would reach the adapter as a missing /// value rather than the empty list dbt was handed. -fn emit_value(out: &mut String, indent: usize, v: &Value) { +/// +/// `credential` masks scalars only. A collection under a credential-named key is still +/// walked: dbt-duckdb's `secrets:` list holds the endpoint and scope that say which +/// connection it is, each entry judged by its own key. +fn emit_value(out: &mut String, indent: usize, v: &Value, mask: bool, credential: bool) { match v { Value::Object(m) => { // A null is an optional field the resource form left unset, and dbt validates @@ -815,7 +896,7 @@ fn emit_value(out: &mut String, indent: usize, v: &Value) { } out.push('\n'); for (k, v) in kept { - emit_entry(out, indent + 2, k, v); + emit_entry(out, indent + 2, k, v, mask); } } Value::Array(items) => { @@ -828,9 +909,10 @@ fn emit_value(out: &mut String, indent: usize, v: &Value) { for item in items { out.push_str(&pad); out.push('-'); - emit_value(out, indent + 2, item); + emit_value(out, indent + 2, item, mask, credential); } } + _ if credential => out.push_str(&format!(" {}\n", yaml_scalar(MASKED_CREDENTIAL))), _ => out.push_str(&format!(" {}\n", yaml_value(v))), } } @@ -904,6 +986,7 @@ mod tests { "dbname": "warehouse", "sslmode": "require"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -936,6 +1019,7 @@ mod tests { "password": "p", "dbname": "warehouse"}); let p = render_profile( &KnownAdapter::Redshift.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1136,6 +1220,7 @@ mod tests { "http_path": "/sql/1.0/warehouses/x", "token": "t"}); let p = render_profile( &KnownAdapter::Databricks.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1161,6 +1246,7 @@ mod tests { "root_certificate_pem": "-----BEGIN CERTIFICATE-----\nx\n"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1188,6 +1274,7 @@ mod tests { let plain = json!({"host": "h", "dbname": "d", "sslmode": "require"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &plain, "wm", "prod", @@ -1200,29 +1287,107 @@ mod tests { assert_eq!(p.root_certificate_pem, None); } - // `snowflake_oauth` maps to the Snowflake adapter, but its credential is a - // token, which dbt honors only with `authenticator: oauth`. Forwarding neither - // renders a profile with no credential at all. - #[test] - fn snowflake_oauth_renders_its_token() { - let r = json!({"account_identifier": "acc", "username": "u", "token": "tok", - "database": "db", "warehouse": "wh"}); - let p = render_profile( + fn snowflake(engine: DbtEngine, r: &Value) -> error::Result { + render_profile( &KnownAdapter::Snowflake.into(), - &r, + engine, + r, "wm", "prod", None, None, std::path::Path::new("/tmp/p"), ) - .unwrap(); - assert!( - p.yaml.contains(" authenticator: \"oauth\"\n"), - "{}", - p.yaml - ); - assert!(p.yaml.contains(" token: \"tok\"\n")); + } + + // `snowflake_oauth` carries an access token, which only one `authenticator` + // per engine accepts: the Rust engines refuse `oauth` without client + // credentials, and dbt-snowflake before 1.9 has no `jwt`. + #[test] + fn snowflake_oauth_names_its_token_per_engine() { + let r = json!({"account_identifier": "acc", "token": "tok", "database": "db"}); + for (engine, authenticator) in [ + (DbtEngine::DbtCore1x, "oauth"), + (DbtEngine::DbtCore2x, "jwt"), + (DbtEngine::Fusion, "jwt"), + ] { + let p = snowflake(engine, &r).unwrap(); + assert!( + p.yaml + .contains(&format!(" authenticator: \"{authenticator}\"\n")), + "{engine:?}: {}", + p.yaml + ); + assert!(p.yaml.contains(" token: \"tok\"\n")); + } + let err = snowflake( + DbtEngine::DbtCore1x, + &json!({"account_identifier": "acc", "token": "tok"}), + ) + .unwrap_err() + .to_string(); + assert!(err.contains("add `database`"), "{err}"); + } + + // Run identity has to survive a credential rotating, which an OAuth token does + // every few minutes, and still change when the connection moves. + #[test] + fn identity_masks_credentials_but_not_the_connection() { + let rendered = |account: &str, token: &str| { + snowflake( + DbtEngine::DbtCore1x, + &json!({"account_identifier": account, "token": token, "database": "db"}), + ) + .unwrap() + .identity + }; + assert_eq!(rendered("acc", "t1"), rendered("acc", "t2")); + assert_ne!(rendered("acc", "t1"), rendered("other", "t1")); + + // A `dbt_profile` block, whose keys are the adapter's own, nested ones too. + let block = |adapter: KnownAdapter, v: Value| { + render_dbt_profile( + &adapter.into(), + v.as_object().unwrap(), + "wm", + "prod", + None, + None, + std::path::Path::new("/tmp/p"), + ) + .unwrap() + .identity + }; + let bq = |project: &str, key: &str, token_uri: &str| { + block( + KnownAdapter::Bigquery, + json!({"type": "bigquery", "project": project, "dataset": "d", + "keyfile_json": {"client_email": "e", "private_key": key, + "token_uri": token_uri}}), + ) + }; + assert_eq!(bq("p", "k1", "t"), bq("p", "k2", "t")); + assert_ne!(bq("p", "k1", "t"), bq("q", "k1", "t")); + assert_ne!(bq("p", "k1", "t"), bq("p", "k1", "elsewhere")); + // Only scalars are masked: a `secrets:` entry still names its endpoint. + // A rotated access key changes its id with its secret. + let duck = |endpoint: &str, secret: &str| { + block( + KnownAdapter::Duckdb, + json!({"type": "duckdb", "path": "x.duckdb", + "secrets": [{"type": "s3", "endpoint": endpoint, + "key_id": format!("id-{secret}"), "secret": secret}]}), + ) + }; + assert_eq!(duck("s3.a", "k1"), duck("s3.a", "k2")); + assert_ne!(duck("s3.a", "k1"), duck("s3.b", "k1")); + let pg = |pass: &str| { + block( + KnownAdapter::Postgres, + json!({"type": "postgres", "host": "h", "user": "u", "pass": pass}), + ) + }; + assert_eq!(pg("p1"), pg("p2")); } // dbt rejects a BigQuery target with no dataset and a service-account JSON @@ -1233,6 +1398,7 @@ mod tests { let r = json!({"project_id": "p", "client_email": "e", "private_key": "k"}); let err = render_profile( &KnownAdapter::Bigquery.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1245,6 +1411,7 @@ mod tests { assert!(err.contains("profile.schema"), "{err}"); let p = render_profile( &KnownAdapter::Bigquery.into(), + DbtEngine::DbtCore1x, &r, "wm", "prod", @@ -1284,6 +1451,7 @@ mod tests { let r = json!({"host": "h", "dbname": "sales", "user": "u"}); let p = render_profile( &KnownAdapter::Mysql.into(), + DbtEngine::DbtCore1x, &r, "wm", "dev", @@ -1304,6 +1472,7 @@ mod tests { fn a_profile_name_or_target_cannot_open_a_sibling_key() { let rendered = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &serde_json::json!({"host": "h", "user": "u", "password": "p", "dbname": "d"}), "prod # hidden", "dev\n evil: yes", @@ -1339,6 +1508,7 @@ mod tests { "password": "p\"\nhost: evil.example.com\n#"}); let p = render_profile( &KnownAdapter::Postgres.into(), + DbtEngine::DbtCore1x, &r, "wm", "dev", diff --git a/backend/windmill-worker/src/dbt_state.rs b/backend/windmill-worker/src/dbt_state.rs new file mode 100644 index 0000000000..ecf3d0dbe3 --- /dev/null +++ b/backend/windmill-worker/src/dbt_state.rs @@ -0,0 +1,748 @@ +//! The dbt state a project last built into one environment, and the state +//! directory a run reads it back through. +//! +//! `dbt --defer --state ` resolves a `ref()` the run does not build to the +//! relation the manifest in `` names, instead of to the schema this run +//! writes into. That makes the state a durable, per-environment artifact rather +//! than a cache: the next run of a project usually lands on a worker holding +//! neither the manifest nor the results, so anything worker-local answers for +//! one machine's history rather than for the environment. +//! +//! Two artifacts live in that directory and both are stored: `manifest.json`, +//! which is what a deferral resolves through, and `run_results.json`, which +//! `select`'s `result:` selectors read — and `select` reaches dbt verbatim, so a +//! state directory missing it fails a selection a user may legitimately write. + +use std::path::{Path, PathBuf}; + +use uuid::Uuid; +use windmill_common::error::{self, Error}; +use windmill_common::worker::Connection; + +use crate::dbt_executor::{digest, PreparedProject, ARTIFACTS_DIR}; + +lazy_static::lazy_static! { + /// Above this, an artifact goes to the instance's object storage instead of + /// into the row. A manifest passes a few hundred KB on a handful of models + /// and grows with the project, so this ceiling is what decides whether a + /// large project needs storage configured at all; a small one stays in the + /// database, where it costs no round trip and needs nothing configured. + static ref DBT_STATE_INLINE_MAX_BYTES: usize = std::env::var("DBT_STATE_INLINE_MAX_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8 * 1024 * 1024); +} + +/// The directory `--state` points at. Inside the job directory, so it sits in +/// the sandbox's one writable bind and goes away with the job, and prefixed like +/// the artifacts directory beside it so a project carrying a directory of this +/// name is not overwritten. +/// +/// Passed to dbt RELATIVE, and that is load-bearing rather than tidiness. dbt +/// records the invocation's flags into `run_results.json` and a later +/// `dbt retry` restores them, so an absolute path would name the job directory +/// of the run being resumed — gone by then, leaving the retry to resolve a +/// deferred `ref()` against nothing. Relative, it resolves against the project +/// root, which is whichever job directory the retry landed in. +pub(crate) const STATE_DIR: &str = "wm_dbt_state"; + +/// Where this run's relations live, which is the only thing a deferral is about. +pub(crate) fn environment(p: &PreparedProject) -> String { + environment_key( + p.warehouse.as_deref(), + // The target dbt RUNS, not the descriptor's: it falls back to the + // workspace warehouse's and to the project's own default, so reading the + // descriptor's would put two inherited targets under one empty name. + p.effective_target.as_deref(), + // The pair `relation_root` reports to the graph's drift check, taken + // apart so neither can absorb the other's delimiter below. + p.default_schema.as_deref(), + p.default_database.as_deref(), + ) +} + +/// The warehouse and the target name the environment; the database and schema +/// they resolve to are in the key because a repointed warehouse resource or a +/// moved schema keeps both names while putting the relations somewhere else — +/// and a manifest is a list of relation names, so a deferral has no other way to +/// notice. A move therefore reads as an environment nothing has published yet. +/// +/// Length-prefixed rather than joined on a separator. Every component but the +/// warehouse is spelled by the user — a dbt target name and a schema are both +/// arbitrary strings a profile may quote — so `prod|analytics` + `scratch` and +/// `prod` + `analytics|scratch` would otherwise be one key, and a profile moving +/// between them would read as the same environment rather than as one nothing +/// has published. Same reasoning as `stable_digest`, and still legible in a row: +/// `4:main|4:prod|9:analytics|12:dbt_wh_defer`. What a MESSAGE names is +/// `environment_label`, since this encoding is for storage. +fn environment_key( + warehouse: Option<&str>, + target: Option<&str>, + schema: Option<&str>, + database: Option<&str>, +) -> String { + [warehouse, target, schema, database] + .iter() + .map(|v| { + let v = v.unwrap_or(""); + format!("{}:{v}", v.len()) + }) + .collect::>() + .join("|") +} + +/// The environment as a message names it: the key above is length-prefixed for +/// storage, which is not something to put in front of a caller. +pub(crate) fn environment_label(p: &PreparedProject) -> String { + format!( + "warehouse `{}`, target `{}`, relations in `{}`", + p.warehouse.as_deref().unwrap_or("(none)"), + p.effective_target + .as_deref() + .unwrap_or("(the profile's default)"), + match (p.default_database.as_deref(), p.default_schema.as_deref()) { + (Some(db), Some(schema)) => format!("{db}.{schema}"), + (None, Some(schema)) => schema.to_string(), + _ => "(the adapter's default)".to_string(), + } + ) +} + +/// The state one environment last published. +pub(crate) struct StoredState { + pub manifest: String, + pub run_results: Option, + /// The run that published it, so a deferring run can say what it deferred to. + pub job_id: Uuid, +} + +/// Publish this run's artifacts as the environment's state. +/// +/// Called for a run that BUILT what the script's own descriptor selects and +/// succeeded (see `handle_dbt_job`). Best-effort in the same sense as the retry +/// state: losing it costs the next deferral, not the run that just finished. +/// +/// **What the artifacts may carry follows from that condition.** A publishing run +/// added nothing of its own — no `select` or `vars` override, and a descriptor +/// interpolating a `{{ }}` placeholder into `vars` never publishes at all — so +/// dbt's `run_results.json` records the descriptor's own arguments, which are the +/// script's content. That is why this is keyed by environment where +/// `dbt_run_state` is keyed by principal: the retry state holds whatever a caller +/// submitted, this holds what the script says. Widen the publish condition and +/// that stops being true. +pub(crate) async fn publish( + p: &PreparedProject, + w_id: &str, + job_id: &Uuid, + // The version this job ran. `None` for a preview, which publishes nothing. + script_hash: Option, + // A build recovered by the automatic in-job node retry has a + // `run_results.json` naming only the nodes that retry redid. The manifest is + // unaffected — it is a function of the project, not of what ran — so the + // state is published without results rather than with a set describing some + // other slice of the build. + results_are_partial: bool, + conn: &Connection, +) -> error::Result<()> { + let Connection::Sql(db) = conn else { + // An agent worker reaches the database only through the API, which does + // not expose this table. + return Ok(()); + }; + if p.script_path.is_empty() { + // A preview has no path to key state on, and an empty one would be + // shared by every dbt script in the workspace. + return Ok(()); + } + if p.templated_location { + // Refused on this side too, not only where a deferral reads. A template + // renders to one location per environment while the key sees the + // template, so publishing would file this run's manifest under a key a + // literal profile shares — and de-templating later would make that stale + // manifest readable as the new location's. + return Ok(()); + } + let artifacts = p.project_dir.join(ARTIFACTS_DIR); + // The manifest is what a deferral resolves through, so there is no state + // without one. Every engine writes it beside the results of a build, so this + // is the invocation that built nothing rather than a case to report. + let Ok(manifest) = tokio::fs::read_to_string(artifacts.join("manifest.json")).await else { + return Ok(()); + }; + let run_results = match results_are_partial { + true => None, + false => tokio::fs::read_to_string(artifacts.join("run_results.json")) + .await + .ok(), + }; + let environment = environment(p); + // Uploaded BEFORE the transaction, and to this publication's own keys, so two + // publishers cannot collide on them and nothing here can overwrite an + // artifact a committed row still names. A failure below has only its own + // objects to drop. + let nonce = Uuid::new_v4(); + let (manifest, manifest_key) = store( + manifest, + "manifest.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await?; + let (run_results, run_results_key) = match run_results { + Some(r) => match store( + r, + "run_results.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await + { + Ok(stored) => stored, + Err(e) => { + forget_objects(&[manifest_key, None]).await; + return Err(e); + } + }, + None => (None, None), + }; + let mine = [manifest_key.clone(), run_results_key.clone()]; + // One publisher per environment at a time, so the row and the objects it + // displaces are settled by one of them at a time. An advisory lock rather + // than the row's, because the first publish of an environment has no row to + // lock and is exactly when two runs of a newly deployed script are most + // likely to race. + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + forget_objects(&mine).await; + return Err(e.into()); + } + }; + let staged = async { + sqlx::query_scalar!( + "SELECT pg_advisory_xact_lock($1)", + publication_lock(w_id, &p.script_path, &environment) + ) + .execute(&mut *tx) + .await?; + // The script row FIRST, and held, so a rename, archive or delete of this + // path either waits for this publication or is seen by it. Reading it + // unlocked leaves a window where lifecycle cleanup finds no row to clear, + // finishes, and this transaction then commits state at a path a new + // script goes on to occupy. Script row before sidecar is also the order + // every other dbt writer takes, which is what keeps the two off a + // deadlock. + // + // The version, not just the path: "some live dbt script is here" is also + // satisfied by a script created at a path this one was renamed away from. + // A preview names no version, so `script_hash` is NULL and nothing + // matches — right for a run of content that was never deployed. + let owns_path = sqlx::query_scalar!( + "SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false AND language = 'dbt' + AND (hash = $3 OR $3 = ANY(parent_hashes)) + FOR SHARE", + w_id, + &p.script_path, + script_hash, + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !owns_path { + return error::Result::Ok(None); + } + // What the row points at NOW, so those objects can go once this one is + // committed in their place — never before, since a reader that has + // already read the row is about to fetch them. + let displaced = sqlx::query!( + "SELECT manifest_key, run_results_key FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(&mut *tx) + .await? + .map(|r| [r.manifest_key, r.run_results_key]) + .unwrap_or_default() + // Never a key this publication is about to commit. The keys carry a + // per-execution nonce so the two cannot coincide, and this is what says + // so rather than leaving it to be re-derived. + .map(|k| k.filter(|k| !mine.iter().flatten().any(|m| m == k))); + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest, manifest_key, run_results, + run_results_key, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now()) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET + job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest, + manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results, + run_results_key = EXCLUDED.run_results_key, updated_at = now()", + w_id, + &p.script_path, + environment, + job_id, + manifest, + manifest_key, + run_results, + run_results_key, + ) + .execute(&mut *tx) + .await?; + error::Result::Ok(Some(displaced)) + } + .await; + let displaced = match staged { + // Refused by the guard, or the write failed: nothing is committed and + // what was uploaded above has no row naming it. + Ok(None) | Err(_) => { + forget_objects(&mine).await; + return staged.map(|_| ()); + } + Ok(Some(displaced)) => displaced, + }; + // A commit that reports an error may still have committed — what was lost can + // be the acknowledgement. Dropping this run's objects would then leave the + // committed row naming objects that are gone, and every deferral would fail + // until the next publication; an orphan costs storage instead. + tx.commit().await?; + forget_objects(&displaced).await; + Ok(()) +} + +/// The environment's state, or `None` where nothing has published one. +pub(crate) async fn load( + p: &PreparedProject, + w_id: &str, + conn: &Connection, +) -> error::Result> { + let Connection::Sql(db) = conn else { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the dbt state stored for this environment, which \ + an agent worker cannot read: it reaches the database only through the API. Run this \ + script on a worker of the main group, or without `defer`" + .to_string(), + )); + }; + let environment = environment(p); + // A publication committing between the row and the objects it named drops + // those objects, so a miss is re-read rather than reported as a state that is + // not there. Re-read for as long as the row keeps MOVING: a reader takes no + // lock, so back-to-back publications can each overtake it, and a fixed one + // retry would report the second as missing. An unmoved row is the other + // answer — nothing republished, so the object really is gone. + let mut tried: Option<(Uuid, Option, Option)> = None; + for _ in 0..PUBLICATIONS_OUTRUN { + let Some(row) = sqlx::query!( + "SELECT job_id, manifest, manifest_key, run_results, run_results_key + FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + let seen = ( + row.job_id, + row.manifest_key.clone(), + row.run_results_key.clone(), + ); + let fetched = async { + let manifest = fetch(row.manifest, row.manifest_key).await?; + let run_results = fetch(row.run_results, row.run_results_key).await?; + error::Result::Ok((manifest, run_results)) + } + .await; + match fetched { + Ok((Some(manifest), run_results)) => { + return Ok(Some(StoredState { manifest, run_results, job_id: seen.0 })) + } + Ok((None, _)) => return Ok(None), + Err(e) => { + if tried.as_ref() == Some(&seen) { + return Err(e); + } + tried = Some(seen); + } + } + } + Err(Error::internal_err( + "the dbt state for this environment was replaced faster than it could be read; run this \ + script again" + .to_string(), + )) +} + +/// How many publications a read may lose to before it gives up. Each one costs a +/// re-read, and a project publishing this often while another run defers is +/// already contending for the same relations. +const PUBLICATIONS_OUTRUN: usize = 5; + +/// A `manifest.json` for a state directory, whichever side it comes from. +/// +/// One enum because the three restores — a deferral's stored state, a retry's +/// worker-local generation, a retry's database row — differ only in where the +/// bytes are, and a second copy of the directory layout is a second chance for +/// one of them to write a directory dbt reads differently. +pub(crate) enum StateManifest { + Bytes(String), + /// A file on this worker, copied rather than read into memory: a manifest + /// grows with the project. + CopyOf(PathBuf), + None, +} + +/// Write the artifacts a dbt state directory holds, creating it if needed. +/// +/// Returns whether a `manifest.json` ended up there — a worker-local generation +/// can be pruned out from under a restore, and the caller then owes a +/// `dbt parse` for one. +pub(crate) async fn write_state_dir( + dir: &Path, + run_results: Option<&str>, + manifest: StateManifest, +) -> error::Result { + tokio::fs::create_dir_all(dir) + .await + .map_err(|e| Error::internal_err(format!("preparing the dbt state directory: {e}")))?; + if let Some(run_results) = run_results { + tokio::fs::write(dir.join("run_results.json"), run_results) + .await + .map_err(|e| Error::internal_err(format!("writing run_results.json: {e}")))?; + } + Ok(match manifest { + StateManifest::Bytes(m) => { + tokio::fs::write(dir.join("manifest.json"), m) + .await + .map_err(|e| Error::internal_err(format!("writing manifest.json: {e}")))?; + true + } + StateManifest::CopyOf(from) => tokio::fs::copy(from, dir.join("manifest.json")) + .await + .is_ok(), + StateManifest::None => false, + }) +} + +/// The advisory lock one environment's publishers take, so only one of them +/// settles the row and the objects it displaces at a time. +/// +/// Derived from the same three components as the row's key. Two environments +/// whose digests collide wait for each other, which costs a moment and nothing +/// else. +fn publication_lock(w_id: &str, script_path: &str, environment: &str) -> i64 { + let d = digest(&format!("{w_id}|{script_path}|{environment}")); + // Parsed unsigned and reinterpreted: half of all digests set the top bit, + // and read as `i64` those overflow and would collapse onto one key. + u64::from_str_radix(&d[..16], 16).unwrap_or_default() as i64 +} + +/// The object-storage key an artifact takes. +/// +/// One key per PUBLICATION, so an upload never overwrites an artifact the +/// committed row still names: a run that fails between its two uploads, or +/// between them and its row, leaves the state pointing at the pair it already +/// had. The row switches to these in one statement and the objects it displaced +/// are dropped afterwards. The path and environment are only a prefix — the row +/// is what says where an artifact is, so state that moves with a renamed script +/// keeps naming objects under the old one. Digested because a Windmill path and a +/// schema name may both carry characters an object key gives meaning to. +/// +/// The `nonce` is per EXECUTION rather than per job, because zombie recovery +/// re-runs a job under its own id: keyed on that alone, the second attempt would +/// overwrite the objects the first attempt's committed row still names, and then +/// read those same keys back as displaced and drop them. +fn object_key( + w_id: &str, + script_path: &str, + environment: &str, + job_id: &Uuid, + nonce: &Uuid, + artifact: &str, +) -> String { + format!( + "wmill_dbt_state/{w_id}/{}/{job_id}.{nonce}/{artifact}", + digest(&format!("{script_path}|{environment}")) + ) +} + +/// Put an artifact where its size says it belongs: `(inline, key)`, exactly one +/// of which is set. +#[allow(clippy::too_many_arguments)] +async fn store( + value: String, + artifact: &str, + environment: &str, + script_path: &str, + w_id: &str, + job_id: &Uuid, + nonce: &Uuid, +) -> error::Result<(Option, Option)> { + if value.len() <= *DBT_STATE_INLINE_MAX_BYTES { + return Ok((Some(value), None)); + } + let key = object_key(w_id, script_path, environment, job_id, nonce, artifact); + let size = value.len(); + if put_object(&key, value).await? { + return Ok((None, Some(key))); + } + Err(Error::BadRequest(format!( + "this project's {artifact} is {}, past the {} this instance keeps in the database, and \ + this instance has no object storage configured to hold it. Configure instance object \ + storage, or raise DBT_STATE_INLINE_MAX_BYTES", + mib(size), + mib(*DBT_STATE_INLINE_MAX_BYTES), + ))) +} + +fn mib(bytes: usize) -> String { + format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0)) +} + +/// Read an artifact back from whichever home the row names. +async fn fetch(inline: Option, key: Option) -> error::Result> { + match (inline, key) { + (Some(inline), _) => Ok(Some(inline)), + (None, Some(key)) => get_object(&key).await.map(Some), + (None, None) => Ok(None), + } +} + +/// Drop the objects nothing points at any more. Best-effort: an object left +/// behind costs storage, and there is nothing useful to do about it in the path +/// of a run that has already finished. +async fn forget_objects(keys: &[Option; 2]) { + for key in keys.iter().flatten() { + delete_object(key).await; + } +} + +/// Whether the artifact was stored. `false` means this instance has no object +/// storage to put it in. +/// +/// The INSTANCE store, where every other internal worker artifact lives — bun +/// bundles, python wheels, job logs, the global cache. Not the workspace's: +/// that bucket is the one workspace members read and write through +/// `job_helpers/*` and `wmill.write_s3_file`, so a manifest there is one any +/// member could replace, and the next deferring run would hand dbt an +/// attacker-chosen `defer_relation` for every unbuilt `ref()` while holding the +/// script's warehouse credentials. Its compiled SQL would be readable there too, +/// for a project the reader may have no access to. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn put_object(key: &str, value: String) -> error::Result { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return Ok(false); + }; + store + .put(&ObjectPath::from(key), bytes::Bytes::from(value).into()) + .await + .map_err(|e| Error::internal_err(format!("storing the dbt state at {key}: {e:#}")))?; + Ok(true) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn get_object(key: &str) -> error::Result { + let Some(store) = windmill_object_store::get_object_store().await else { + return Err(missing_storage()); + }; + let bytes = windmill_object_store::attempt_fetch_bytes(store, key).await?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::internal_err(format!("the stored dbt state is not valid UTF-8: {e}"))) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn delete_object(key: &str) { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return; + }; + if let Err(e) = store.delete(&ObjectPath::from(key)).await { + tracing::warn!("dbt: could not drop the superseded state object {key}: {e:#}"); + } +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn delete_object(_key: &str) {} + +/// A build without the instance store carries no client at all, so an oversized +/// artifact has nowhere but the row, and a row naming a key was written by a +/// worker that did have one. +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn put_object(_key: &str, _value: String) -> error::Result { + Ok(false) +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn get_object(_key: &str) -> error::Result { + Err(missing_storage()) +} + +fn missing_storage() -> Error { + Error::BadRequest( + "the dbt state for this environment is in the instance's object storage, which this \ + worker cannot reach: it is no longer configured, or this worker was built without \ + object-storage support" + .to_string(), + ) +} + +/// The stored state this run resolves its unbuilt `ref()`s through, materialised +/// into the job directory at `STATE_DIR`. +#[derive(Clone, Debug)] +pub(crate) struct Deferral { + /// The run that published the state, so the job log and the result can say + /// what this one deferred to. + pub published_by: Uuid, + /// Whether the state carries `run_results.json` beside its manifest. A build + /// recovered by node retry publishes without one, and that is the only file a + /// `result:` selector reads. + pub has_run_results: bool, +} + +/// Materialise the environment's state so `--state` has a directory to read. +/// +/// Refused rather than run without deferral where nothing is published: the run +/// would build against a `ref()` resolving into the schema it writes, and fail +/// deep inside dbt with a relation-not-found the caller has no way to connect +/// back to a missing state. +pub(crate) async fn prepare_deferral( + p: &PreparedProject, + w_id: &str, + job_dir: &str, + conn: &Connection, +) -> error::Result { + if p.script_path.is_empty() { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the state a previous run of this script \ + published, so it needs a deployed script; a preview run has no environment to have \ + published one" + .to_string(), + )); + } + // An environment is the warehouse, the target and where they RESOLVE to, and + // a `profiles.yml` that templates its schema or database resolves somewhere + // this runtime does not render. Two renderings would then share one + // environment, and a deferral after the value changed would resolve every + // unbuilt `ref()` through the previous location's manifest. + if p.templated_location { + return Err(Error::BadRequest( + "this project's profile selects its schema or database with a template, which dbt \ + renders and Windmill does not — so two environments cannot be told apart and a \ + deferral could resolve through the wrong one's manifest. Spell the target's schema \ + and database literally to use `defer`" + .to_string(), + )); + } + let Some(state) = load(p, w_id, conn).await? else { + return Err(Error::BadRequest(format!( + "no dbt state is stored for this environment ({}), so a `ref()` this run does not \ + build has no relation to resolve to. It is published by a successful run that adds \ + nothing of its own: one overriding `select` or `vars` does not publish, and neither \ + does any run of a descriptor that interpolates a `{{{{ }}}}` placeholder into `vars` \ + or a `$var:` into `env` — those describe a model set the caller's arguments decided. \ + Run this script once without `defer` and without overrides", + environment_label(p) + ))); + }; + let has_run_results = state.run_results.is_some(); + write_state_dir( + &PathBuf::from(job_dir).join(STATE_DIR), + state.run_results.as_deref(), + StateManifest::Bytes(state.manifest), + ) + .await?; + Ok(Deferral { published_by: state.job_id, has_run_results }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Every component earns its place: a deferral resolves relation NAMES, so + // state published where those names meant something else has to read as no + // state at all rather than as state that silently no longer fits. + #[test] + fn a_moved_profile_is_another_environment() { + let here = environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")); + assert_eq!( + here, + environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("other"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("dev"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("prod"), Some("marts"), Some("wh")) + ); + assert_ne!( + here, + environment_key( + Some("main"), + Some("prod"), + Some("analytics"), + Some("other_db") + ) + ); + } + + // A target name and a schema are both the user's own strings, so a component + // carrying the separator must not be able to spell another tuple's key: a + // profile moving between the two would read as the same environment and + // defer through the manifest of relations that are somewhere else. + #[test] + fn a_component_cannot_spell_another_environments_key() { + assert_ne!( + environment_key(Some("main"), Some("prod|analytics"), Some("scratch"), None), + environment_key(Some("main"), Some("prod"), Some("analytics|scratch"), None) + ); + assert_ne!( + environment_key(Some("main"), Some("prod"), Some("a"), Some("b|c")), + environment_key(Some("main"), Some("prod"), Some("a|b"), Some("c")) + ); + // A component the profile leaves out is the same environment as one it + // spells empty: there is no target named "". + assert_eq!( + environment_key(Some("main"), None, Some("a"), None), + environment_key(Some("main"), Some(""), Some("a"), Some("")) + ); + } + + // Two environments must not queue behind one advisory lock, which is what a + // digest folded through a signed parse did for every one whose top bit is + // set — half of them. + #[test] + fn each_environment_gets_its_own_publication_lock() { + let mut seen = std::collections::HashSet::new(); + for i in 0..64 { + seen.insert(publication_lock( + "ws", + "f/a/p", + &format!("main|prod|s{i}|db"), + )); + } + assert_eq!(seen.len(), 64); + assert_eq!( + publication_lock("ws", "f/a/p", "e"), + publication_lock("ws", "f/a/p", "e") + ); + } +} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..aecdb738b3 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -13,8 +13,8 @@ use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ - get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked, - strip_fork_reserved_attach_args, DucklakeCatalogResourceType, + get_datatable_resource_from_db, get_ducklake_from_db_unchecked, + strip_fork_reserved_attach_args, DatatableAccess, DucklakeCatalogResourceType, }; use windmill_common::PgDatabase; use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE; @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, @@ -1492,13 +1494,9 @@ pub async fn do_duckdb( .await? { probe_blocks.extend(q); - } else if let Some(q) = transform_attach_datatable( - &query_block, - conn, - &mut hidden_passwords, - &job.workspace_id, - ) - .await? + } else if let Some(q) = + transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job) + .await? { probe_blocks.extend(q); } else { @@ -1573,13 +1571,9 @@ pub async fn do_duckdb( .await? { v.extend(ducklake_query); - } else if let Some(datatable_query) = transform_attach_datatable( - &query_block, - conn, - &mut hidden_passwords, - &job.workspace_id, - ) - .await? + } else if let Some(datatable_query) = + transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job) + .await? { v.extend(datatable_query); } else { @@ -2607,33 +2601,79 @@ fn fork_defer_statements( Ok(stmts) } +struct AttachedDatatable<'a> { + /// The data table reference, query string included; a bare `datatable` is `main`. + reference: String, + alias: &'a str, +} + +/// `ATTACH 'datatable[://][?role=]' AS `. A bare `datatable` names the default +/// data table, so the role query string has to be accepted with and without an explicit name. The +/// reference is split only once the workspace can be read, because a stored name may contain `?`. +fn parse_attach_datatable(query: &str) -> Option> { + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new( + r"(?i)ATTACH\s*'datatable(://[^':]+|\?[^':]*)?'\s*AS\s+([^ ;]+)" + ).unwrap(); + } + let cap = RE.captures(query)?; + let reference = match cap.get(1).map(|m| m.as_str()) { + Some(named) if named.starts_with("://") => named[3..].to_string(), + Some(query) => format!("main{query}"), + None => "main".to_string(), + }; + let alias = cap.get(2).map(|m| m.as_str()).unwrap_or(""); + Some(AttachedDatatable { reference, alias }) +} + async fn transform_attach_datatable( query: &str, conn: &Connection, hidden_passwords: &mut Arc>>, - w_id: &str, + job: &MiniPulledJob, ) -> Result>> { - lazy_static::lazy_static! { - static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'datatable(://[^':]+)?'\s*AS\s+([^ ;]+)").unwrap(); - } - let Some(cap) = RE.captures(query) else { + let Some(attached) = parse_attach_datatable(query) else { return Ok(None); }; - let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main"); - let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or(""); + // A query string that does not parse is refused rather than dropped: attaching under the + // default role when the statement asked for another one is the failure this guards. let db_resource = match conn { Connection::Http(client) => { - get_datatable_resource_from_agent_http(client, name, w_id).await? + let (name, role) = + windmill_common::workspaces::parse_datatable_ref(&attached.reference)?; + get_datatable_resource_from_agent_http(client, name, &job.workspace_id, role, &job.id) + .await? + } + Connection::Sql(db) => { + let (name, role) = windmill_common::workspaces::parse_datatable_ref_for( + db, + &job.workspace_id, + &attached.reference, + ) + .await?; + get_datatable_resource_from_db( + db, + &job.workspace_id, + &name, + role.as_deref(), + DatatableAccess::PermissionedAs { + permissioned_as: &job.permissioned_as, + email: &job.permissioned_as_email, + }, + ) + .await? } - Connection::Sql(db) => get_datatable_resource_from_db_unchecked(db, w_id, name).await?, }; if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) { hidden_passwords.lock().unwrap().push(pwd.to_string()); } - Ok(Some(pg_secret_attach_statements(db_resource, alias_name)?)) + Ok(Some(pg_secret_attach_statements( + db_resource, + attached.alias, + )?)) } // Secret names must be plain identifiers; the hash keeps two aliases distinct even @@ -2678,6 +2718,11 @@ fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result' (TYPE postgres, SECRET …)` would reach a database nobody + // authorized this job for, as this role. + format!("DROP TEMPORARY SECRET {secret_name};"), ]) } @@ -2751,6 +2796,45 @@ pub struct Arg { mod tests { use super::*; + #[test] + fn attach_datatable_parses_name_and_role() { + let reference_of = |q: &str| parse_attach_datatable(q).unwrap().reference; + let named = + parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt").unwrap(); + assert_eq!( + (named.reference.as_str(), named.alias), + ("sales?role=analytics", "dt") + ); + // A bare `datatable` is the default one, and still takes a role. + assert_eq!( + reference_of("ATTACH 'datatable?role=analytics' AS dt"), + "main?role=analytics" + ); + assert_eq!(reference_of("ATTACH 'datatable://sales' AS dt"), "sales"); + assert_eq!(reference_of("ATTACH 'datatable' AS dt"), "main"); + assert!(parse_attach_datatable("SELECT 1").is_none()); + // A stored name can contain `?`, so that is left to the workspace lookup to split. + assert_eq!(reference_of("ATTACH 'datatable://a?b' AS dt"), "a?b"); + + // The key matches case-insensitively, as the `-- role` annotation does, and a query string + // that does not parse is refused rather than attached under the default role. + let parse = |q: &str| { + windmill_common::workspaces::parse_datatable_ref(&reference_of(q)) + .map(|(name, role)| (name.to_string(), role.map(str::to_string))) + }; + assert_eq!( + parse("ATTACH 'datatable://sales?Role=analytics' AS dt").unwrap(), + ("sales".to_string(), Some("analytics".to_string())) + ); + for malformed in [ + "ATTACH 'datatable://sales?role=' AS dt", + "ATTACH 'datatable://sales?role=an;alytics' AS dt", + "ATTACH 'datatable://sales?x=1&role=analytics' AS dt", + ] { + assert!(parse(malformed).is_err(), "silently ignored: {malformed}"); + } + } + #[test] fn decode_ffi_error_unescapes_multiline_and_strips_quotes() { // Mirror the FFI: JSON-encode the raw DuckDB message, prefix "ERROR ". @@ -3866,6 +3950,8 @@ mod tests { stmts[3], format!("ATTACH 'sslmode=require' AS dt (TYPE postgres, SECRET {secret_name});") ); + assert_eq!(stmts[4], format!("DROP TEMPORARY SECRET {secret_name};")); + assert_eq!(stmts.len(), 5); } #[test] diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 5e9666722a..1c3bbf870f 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -9,6 +9,73 @@ use std::sync::Arc; pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std::env::consts::ARCH); +#[cfg(all(feature = "enterprise", feature = "parquet"))] +lazy_static::lazy_static! { + /// Object-store clients are built with their request timeout disabled so a large job + /// payload can stream for as long as it needs. A cache transfer must not inherit that: + /// a put or get that stalls after connecting would otherwise hold the job for its whole + /// duration limit, with nothing in the job log saying why. + pub(crate) static ref OBJECT_STORE_CACHE_IO_TIMEOUT: std::time::Duration = std::time::Duration::from_secs( + std::env::var("OBJECT_STORE_CACHE_IO_TIMEOUT_SECS") + .ok() + .and_then(|x| x.parse::().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(30 * 60), + ); +} + +/// A cache transfer that did not complete: the store answered with an error, or +/// [`OBJECT_STORE_CACHE_IO_TIMEOUT`] ran out first. Nothing is logged on the way out: a failed +/// download is usually an ordinary miss, so each call site decides which outcome gets a line, +/// and logs it once. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub(crate) enum CacheIoError { + TimedOut { what: String, path: String, secs: u64 }, + Failed(error::Error), +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +impl std::fmt::Display for CacheIoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CacheIoError::TimedOut { what, path, secs } => write!( + f, + "{what} {path} in the object store cache timed out after {secs}s \ + (OBJECT_STORE_CACHE_IO_TIMEOUT_SECS)" + ), + CacheIoError::Failed(e) => write!(f, "{e:#}"), + } + } +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +impl From for error::Error { + fn from(e: CacheIoError) -> Self { + match e { + CacheIoError::Failed(e) => e, + timed_out => error::Error::ExecutionErr(timed_out.to_string()), + } + } +} + +/// Runs one object-store cache transfer under [`OBJECT_STORE_CACHE_IO_TIMEOUT`]. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +pub(crate) async fn bounded_cache_io( + what: &str, + path: &str, + io: impl std::future::Future>, +) -> Result { + let timeout = *OBJECT_STORE_CACHE_IO_TIMEOUT; + match tokio::time::timeout(timeout, io).await { + Ok(result) => result.map_err(CacheIoError::Failed), + Err(_) => Err(CacheIoError::TimedOut { + what: what.to_string(), + path: path.to_string(), + secs: timeout.as_secs(), + }), + } +} + #[cfg(all(feature = "enterprise", feature = "parquet"))] pub async fn build_tar_and_push( s3_client: Arc, @@ -56,17 +123,20 @@ pub async fn build_tar_and_push( // let s3_client = s3_settings.as_ref().ok_or_else(|| { // error::Error::ExecutionErr("Failed to read s3 cache settings".to_string()) // })?; - if let Err(e) = s3_client - .put( - &Path::from(format!( - "/tar/{}/{lang}/{folder_name}.tar", - if platform_agnostic { "" } else { TARGET } - )), - std::fs::read(&tar_path)?.into(), - ) - .await - { - tracing::info!("Failed to put tar to s3: {tar_path}. Error: {:?}", e); + let remote_path = format!( + "/tar/{}/{lang}/{folder_name}.tar", + if platform_agnostic { "" } else { TARGET } + ); + let tar_bytes = std::fs::read(&tar_path)?; + let put = bounded_cache_io("uploading", &remote_path, async { + s3_client + .put(&Path::from(remote_path.as_str()), tar_bytes.into()) + .await + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e) = put { + tracing::info!("Failed to put tar to s3: {tar_path}. Error: {e}"); return Err(error::Error::ExecutionErr(format!( "Failed to put tar to s3: {tar_path}" ))); @@ -109,7 +179,12 @@ pub async fn pull_from_tar( "tar/{}/{lang}/{folder_name}.tar", if platform_agnostic { "" } else { TARGET } ); - let bytes = attempt_fetch_bytes(client, &tar_path).await?; + let bytes = bounded_cache_io( + "downloading", + &tar_path, + attempt_fetch_bytes(client, &tar_path), + ) + .await?; extract_tar(bytes, &folder).map_err(|e| { tracing::error!("Failed to extract piptar {folder_name}. Error: {:?}", e); @@ -147,17 +222,29 @@ pub fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { Ok(()) } -/// Two-tier cache load: check local disk first, then fall back to instance object store. +/// Two-tier cache load: check local disk first, then fall back to the shared object store. +/// +/// "Shared" is the worker group's own store when its config overrides one, the instance store +/// otherwise — see [`windmill_object_store::get_cache_object_store`]. /// Returns `(hit, log_message)`. pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let started = std::time::Instant::now(); - if let Ok(mut x) = windmill_object_store::attempt_fetch_bytes(os, _remote_path).await { + let fetched = bounded_cache_io( + "downloading", + _remote_path, + windmill_object_store::attempt_fetch_bytes(os, _remote_path), + ) + .await; + if let Err(e @ CacheIoError::TimedOut { .. }) = &fetched { + tracing::error!("{e}"); + } + if let Ok(mut x) = fetched { if is_dir { // Extract into a sibling temp dir then atomically publish it, // so a concurrent cold-load gating on metadata(bin_path) never @@ -200,13 +287,15 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bo } } -/// Whether this worker can push to the instance object store at all — the features are +/// Whether this worker can push to the shared object store at all — the features are /// compiled in and a store is loaded. False on builds without them, where `save_cache` /// only ever writes to the worker's own disk. pub async fn object_store_available() -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] { - windmill_object_store::get_object_store().await.is_some() + windmill_object_store::get_cache_object_store() + .await + .is_some() } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] { @@ -214,20 +303,26 @@ pub async fn object_store_available() -> bool { } } -/// Whether a binary/bundle is in the instance object store, ignoring the local cache. +/// Whether a binary/bundle is in the shared object store, ignoring the local cache. /// /// The deploy-time prebuild asks this rather than [`exists_in_cache`]: a copy on the /// building worker's own disk is exactly the state the prebuild exists to fix, so /// answering from it would latch a failed upload into a permanent skip. pub async fn exists_in_object_store(_remote_path: &str) -> bool { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { - return os - .head(&windmill_object_store::object_store_reexports::Path::from( + if let Some(os) = windmill_object_store::get_cache_object_store().await { + let head = bounded_cache_io("checking", _remote_path, async { + os.head(&windmill_object_store::object_store_reexports::Path::from( _remote_path, )) .await - .is_ok(); + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e @ CacheIoError::TimedOut { .. }) = &head { + tracing::error!("{e}"); + } + return head.is_ok(); } false } @@ -241,30 +336,36 @@ pub async fn ensure_pushed_to_object_store(remote_path: &str) -> error::Result<( return Ok(()); } Err(error::Error::ExecutionErr(format!( - "the binary was built but did not reach the instance object store at {remote_path}, \ + "the binary was built but did not reach the object store at {remote_path}, \ so no other worker can load it" ))) } -/// Check whether a binary/bundle exists in local cache or instance object store. +/// Check whether a binary/bundle exists in local cache or the shared object store. pub async fn exists_in_cache(bin_path: &str, _remote_path: &str) -> bool { if tokio::fs::metadata(&bin_path).await.is_ok() { return true; } else { #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { - return os - .get(&windmill_object_store::object_store_reexports::Path::from( + if let Some(os) = windmill_object_store::get_cache_object_store().await { + let get = bounded_cache_io("checking", _remote_path, async { + os.get(&windmill_object_store::object_store_reexports::Path::from( _remote_path, )) .await - .is_ok(); + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e @ CacheIoError::TimedOut { .. }) = &get { + tracing::error!("{e}"); + } + return get.is_ok(); } return false; } } -/// Two-tier cache write: upload to instance object store, then copy to local disk. +/// Two-tier cache write: upload to the shared object store, then copy to local disk. pub async fn save_cache( local_cache_path: &str, _remote_cache_path: &str, @@ -275,7 +376,7 @@ pub async fn save_cache( let mut _cached_to_s3 = false; #[cfg(all(feature = "enterprise", feature = "parquet"))] - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { use windmill_object_store::object_store_reexports::Path; let file_to_cache = if is_dir { let tar_path = format!( @@ -302,17 +403,15 @@ pub async fn save_cache( origin.to_owned() }; - if let Err(e) = os - .put( - &Path::from(_remote_cache_path), - std::fs::read(&file_to_cache)?.into(), - ) - .await - { - tracing::error!( - "Failed to put bin to object store: {_remote_cache_path}. Error: {:?}", - e - ); + let bytes = std::fs::read(&file_to_cache)?; + let put = bounded_cache_io("uploading", _remote_cache_path, async { + os.put(&Path::from(_remote_cache_path), bytes.into()) + .await + .map_err(|e| error::Error::ExecutionErr(format!("{e:?}"))) + }) + .await; + if let Err(e) = put { + tracing::error!("Failed to put bin to object store: {_remote_cache_path}. Error: {e}"); } else { _cached_to_s3 = true; if is_dir { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 1d2009c5c0..9fe6d4a616 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -230,8 +230,15 @@ fn go_runtime_int32(v: &str) -> Option { /// Cache key of a Go build. The run path and the deploy-time prebuild must derive it the /// same way or the prebuilt binary is never found and gets rebuilt on first run. -fn go_cache_key(code: &str, maybe_lock: &MaybeLock) -> String { - calculate_hash(&format!("{}{:?}v2", code, maybe_lock)) +fn go_cache_key( + code: &str, + maybe_lock: &MaybeLock, + modules: Option<&HashMap>, +) -> String { + // A module whose path starts with `go/` lands inside the module dir this builds, so + // its content ends up in the binary the key names. + let base = format!("{}{:?}v2", code, maybe_lock); + crate::worker::artifact_cache_name(base, modules) } /// Install the deps, generate the entrypoint wrapper, `go build`, and push the binary to @@ -459,9 +466,10 @@ pub async fn prebuild_go_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { let maybe_lock = MaybeLock::Resolved { lock: lock.to_string() }; - let hash = go_cache_key(code, &maybe_lock); + let hash = go_cache_key(code, &maybe_lock, modules); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -512,6 +520,7 @@ pub async fn handle_go_job( envs: HashMap, occupation_metrics: &mut OccupancyMetrics, maybe_lock: MaybeLock, + modules: Option<&HashMap>, ) -> Result, Error> { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); @@ -520,7 +529,7 @@ pub async fn handle_go_job( .create(&job_dir) .expect("could not create go job dir"); - let hash = go_cache_key(inner_content, &maybe_lock); + let hash = go_cache_key(inner_content, &maybe_lock, modules); let bin_path = format!("{}/{hash}", *GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index bb0518f22d..2e47fced18 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -66,6 +66,7 @@ pub(crate) struct JobHandlerInput<'a> { pub requirements_o: Option<&'a String>, pub shared_mount: &'a str, pub worker_name: &'a str, + pub modules: Option<&'a HashMap>, } pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result, Error> { @@ -612,25 +613,33 @@ async fn compile<'a>( inner_content, requirements_o, parent_runnable_path, + modules, .. }: &mut JobHandlerInput<'a>, classpath: &'a str, // plugins: Vec<&'a str>, ) -> Result<(), Error> { - fn compute_hash(code: &str, requirements_o: Option<&String>) -> String { - calculate_hash(&format!( + // The cached artifact is the whole `target/` dir, and companion modules are written + // into the job dir before this runs, so their content can land in it. + fn compute_hash( + code: &str, + requirements_o: Option<&String>, + modules: Option<&HashMap>, + ) -> String { + let base = format!( "{}{}", code, requirements_o .as_ref() .map(|x| x.to_string()) .unwrap_or_default() - )) + ); + crate::worker::artifact_cache_name(base, modules) } let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let ws_suffix = crate::workspace_registry_cache_suffix(&job.workspace_id).await; - let mut hash = compute_hash(inner_content, *requirements_o); + let mut hash = compute_hash(inner_content, *requirements_o, *modules); hash.push_str(&ws_suffix); let bin_path = format!("{}/{hash}", *JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index e3e7868548..6059936af6 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -33,9 +33,11 @@ pub mod common; mod config; mod csharp_executor; +mod dbt_column_index; mod dbt_engine; mod dbt_executor; mod dbt_profiles; +mod dbt_state; #[cfg(feature = "private")] mod dedicated_worker_ee; mod dedicated_worker_oss; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 1413924bf3..69aa3ee0fb 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -26,9 +26,11 @@ use windmill_common::azure_workload_identity::WORKLOAD_IDENTITY_PASSWORD; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; use windmill_common::worker::{ - to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, + to_raw_value, Connection, SqlAnnotations, SqlResultCollectionStrategy, CLOUD_HOSTED, +}; +use windmill_common::workspaces::{ + get_datatable_resource_from_db, parse_datatable_ref, parse_datatable_ref_for, DatatableAccess, }; -use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; use windmill_common::{PgDatabase, PrepareQueryColumnInfo, PrepareQueryResult, DB}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ @@ -680,15 +682,36 @@ pub async fn do_postgresql( } else { match pg_args.get("database").cloned() { Some(Value::String(db_str)) if db_str.starts_with("datatable://") => { - let db_str = db_str.trim_start_matches("datatable://"); + let reference = db_str.trim_start_matches("datatable://"); + // The annotation wins: a generated query can carry a `?role=` in the reference it + // was handed, but only the script's author writes the leading comment block. + let annotated = SqlAnnotations::datatable_role(&query)?; Some(match conn { Connection::Http(client) => { - get_datatable_resource_from_agent_http(client, &db_str, &job.workspace_id) - .await? + let (name, uri_role) = parse_datatable_ref(reference)?; + get_datatable_resource_from_agent_http( + client, + name, + &job.workspace_id, + annotated.as_deref().or(uri_role), + &job.id, + ) + .await? } Connection::Sql(db) => { - get_datatable_resource_from_db_unchecked(db, &job.workspace_id, &db_str) - .await? + let (name, uri_role) = + parse_datatable_ref_for(db, &job.workspace_id, reference).await?; + get_datatable_resource_from_db( + db, + &job.workspace_id, + &name, + annotated.as_deref().or(uri_role.as_deref()), + DatatableAccess::PermissionedAs { + permissioned_as: &job.permissioned_as, + email: &job.permissioned_as_email, + }, + ) + .await? } }) } diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 1ede0933cb..03328b82cb 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -14,7 +14,7 @@ use windmill_common::{ }; use windmill_queue::MiniPulledJob; -use windmill_parser::Typ; +use windmill_parser::{MainArgSignature, Typ}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -40,6 +40,43 @@ lazy_static::lazy_static! { const COMPOSER_LOCK_SPLIT: &str = "\nLOCK\n"; +static PHP_PARSER_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +pub(crate) async fn parse_php_signature( + code: &str, + main_override: Option, +) -> Result { + parse_php_signature_with_slot(code, main_override, &PHP_PARSER_SLOT).await +} + +async fn parse_php_signature_with_slot( + code: &str, + main_override: Option, + slot: &'static tokio::sync::Semaphore, +) -> Result { + let acquire = slot.acquire(); + tokio::pin!(acquire); + // Retain the acquisition across the warning to preserve its FIFO queue position. + let permit = tokio::select! { + permit = &mut acquire => permit, + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => { + tracing::warn!("Waiting over a second for PHP signature parser capacity"); + acquire.await + } + } + .map_err(to_anyhow)?; + let code = code.to_owned(); + tokio::task::spawn_blocking(move || { + // Parsing walks the entire AST. Keep its stack off async workers and retain + // the process-wide CPU limit even if the awaiting job is cancelled. + let _permit = permit; + windmill_parser_php::parse_php_signature(&code, main_override) + }) + .await + .map_err(|e| error::Error::internal_err(format!("PHP signature parsing task failed: {e}")))? + .map_err(Into::into) +} + pub fn parse_php_imports(code: &str) -> anyhow::Result> { let find_requirements = code .lines() @@ -333,11 +370,9 @@ pub async fn handle_php_job( let main_override = job.script_entrypoint_override.as_deref(); let write_wrapper_f = async { - let args = windmill_parser_php::parse_php_signature( - inner_content, - main_override.map(ToString::to_string), - )? - .args; + let args = parse_php_signature(inner_content, main_override.map(ToString::to_string)) + .await? + .args; let args_to_include = args .iter() @@ -491,3 +526,51 @@ try {{ .await?; read_result(job_dir, None).await } + +#[cfg(test)] +mod tests { + use super::parse_php_signature_with_slot; + + #[test] + fn cancelled_parse_retains_slot_until_blocking_work_finishes() { + static SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + let runtime = tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let blocker = runtime.spawn_blocking(move || { + started_tx.send(()).unwrap(); + let _ = release_rx.recv(); + }); + + runtime.block_on(async { + started_rx.await.unwrap(); + let parse = tokio::spawn(parse_php_signature_with_slot( + ") { use crate::global_cache::build_tar_and_push; - use windmill_object_store::get_object_store; + use windmill_object_store::get_cache_object_store; while let Some(task) = rx.recv().await { - if let Some(os) = get_object_store().await { + if let Some(os) = get_cache_object_store().await { match build_tar_and_push(os, task.venv_path.clone(), task.cache_dir, None, false).await { Ok(()) => { @@ -117,6 +118,12 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); +/// Every file exchanged with a job is UTF-8 by construction, so the interpreter +/// must agree. A job env carries no locale: Linux then picks UTF-8 on its own +/// (PEP 540), Windows picks the ANSI code page. Applied after the whitelisted +/// envs so the protocol is not the user's to opt out of. +pub const PYTHON_UTF8_ENVS: [(&str, &str); 1] = [("PYTHONUTF8", "1")]; + /// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a /// Python dict literal. Preview jobs pass a path -> temp-hash map so relative /// imports resolve from not-yet-deployed local content; deployed runs pass @@ -138,7 +145,7 @@ pub fn has_relative_imports(content: &str) -> bool { use crate::global_cache::pull_from_tar; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use windmill_object_store::OBJECT_STORE_SETTINGS; +use windmill_object_store::get_cache_object_store; use crate::{ common::{ @@ -221,9 +228,9 @@ fn filter_pip_local_dependencies(lines: Vec) -> (Vec, Vec, compiled_deps: &[Regex]) -> (Vec, Vec) { - let (ignored, kept): (Vec, Vec) = lines - .into_iter() - .partition(|s| !s.starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s))); + let (ignored, kept): (Vec, Vec) = lines.into_iter().partition(|s| { + !s.trim_start().starts_with('#') && compiled_deps.iter().any(|dep| dep.is_match(s)) + }); (kept, ignored) } @@ -317,6 +324,11 @@ pub async fn uv_pip_compile( "compile", "-q", "--no-header", + // The `#`-line filter applied to the output below only catches whole-line + // annotations, and uv's annotation style is configurable: `[pip] + // annotation-style = "line"` in the worker HOME's uv.toml emits them inline + // ("anyio==4.15.1 # via httpx"), which that filter keeps. + "--no-annotate", file, "--strip-extras", "-o", @@ -818,7 +830,7 @@ pub async fn handle_python_job( del pre_args[k] kwargs = inner_script.preprocessor(**pre_args) kwrags_json = res_to_json(kwargs, type(kwargs)) - with open("args.json", 'w') as f: + with open("args.json", 'w', encoding="utf-8") as f: f.write(kwrags_json)"# ) } else { @@ -853,7 +865,7 @@ pub async fn handle_python_job( _pre_result = asyncio.run(_pre_result) kwargs = _pre_result if _pre_result is not None else {{}} _pre_json = json.dumps(kwargs, separators=(',', ':'), default=str) - with open("args.json", 'w') as f: + with open("args.json", 'w', encoding="utf-8") as f: f.write(_pre_json) sys.stdout.write("wm_res[preprocessed_args]:" + _pre_json + "\n") sys.stdout.flush()"# @@ -874,11 +886,11 @@ import sys from {module_dir_dot} import {last} as inner_script from wmill.client import _run_workflow -with open("args.json") as f: +with open("args.json", encoding="utf-8") as f: kwargs = json.load(f, strict=False) {transforms} -with open("checkpoint.json") as f: +with open("checkpoint.json", encoding="utf-8") as f: checkpoint = json.load(f, strict=False) result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") @@ -904,12 +916,12 @@ try: print("") print("--- WAC: complete ---") output_json = json.dumps(output, separators=(',', ':'), default=str) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: f.write(output_json) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} extra = e.__dict__ if extra and len(extra) > 0: @@ -936,7 +948,7 @@ import sys from {module_dir_dot} import {last} as inner_script import re -with open("args.json") as f: +with open("args.json", encoding="utf-8") as f: kwargs = json.load(f, strict=False) args = {{}} {transforms} @@ -972,12 +984,12 @@ try: print("WM_STREAM: " + chunk.replace('\n', '\\n')) res = None res_json = res_to_json(res, typ) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: f.write(res_json) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} extra = e.__dict__ if extra and len(extra) > 0: @@ -1117,6 +1129,7 @@ mount {{ ) .await?, ) + .envs(PYTHON_UTF8_ENVS) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1152,6 +1165,7 @@ mount {{ ) .await?, ) + .envs(PYTHON_UTF8_ENVS) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1223,6 +1237,7 @@ mount {{ result, job, conn, + canceled_by, modules, new_args.as_ref(), )) @@ -2323,6 +2338,33 @@ async fn spawn_uv_install( } } +/// First field (the path) of a wheel RECORD line. RECORD is CSV (PEP 376 / +/// RFC 4180): a path containing a comma or a double quote is written quoted, +/// with inner quotes doubled, so splitting on the first comma turns such an +/// entry into a name that never exists on disk. +fn record_first_field(line: &str) -> Option { + let Some(quoted) = line.strip_prefix('"') else { + return line + .split(',') + .next() + .filter(|p| !p.is_empty()) + .map(str::to_owned); + }; + let mut field = String::new(); + let mut chars = quoted.chars(); + while let Some(c) = chars.next() { + if c != '"' { + field.push(c); + } else if chars.as_str().starts_with('"') { + chars.next(); + field.push('"'); + } else { + return Some(field).filter(|f| !f.is_empty()); + } + } + None +} + /// Verify that every file listed in the wheel's RECORD exists on disk under /// `venv_p`. Used as a structural integrity check after both a successful /// `pull_from_tar` (object-store cache hit) and a successful local @@ -2371,9 +2413,9 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { if trimmed.is_empty() { continue; } - let rel_path = match trimmed.split(',').next() { - Some(p) if !p.is_empty() => p, - _ => continue, + let rel_path = match record_first_field(trimmed) { + Some(p) => p, + None => continue, }; // Defensive: skip absolute paths or escaping entries — we only // validate package-relative files. @@ -2382,7 +2424,7 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { } let full = format!("{venv_p}/{rel_path}"); if tokio::fs::metadata(&full).await.is_err() { - missing.push(rel_path.to_string()); + missing.push(rel_path); // Bound error size in pathological cases (e.g. wholly empty dir). if missing.len() >= 10 { missing.push("...".to_string()); @@ -2440,7 +2482,7 @@ pub async fn handle_python_reqs( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if OBJECT_STORE_SETTINGS.read().await.is_none() { + if get_cache_object_store().await.is_none() { (s3_pull, s3_push) = (false, false); } @@ -2511,11 +2553,23 @@ pub async fn handle_python_reqs( // Find out if there is already cached dependencies // If so, skip them let mut in_cache = vec![]; + if requirements + .iter() + .any(|r| lockfile_line_has_continuation(r)) + { + tracing::warn!(workspace_id = %w_id, job_id = %job_id, "lockfile continues entries across lines; the continued lines are dropped"); + append_logs( + job_id, + w_id, + "\n[!] lockfile continues entries across lines and the continued lines are dropped: `--hash=` pins, extras and markers written that way do not apply\n".to_string(), + conn, + ) + .await; + } for req in &requirements { - // Ignore python version annotation backed into lockfile - if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { + let Some(req) = requirement_from_lockfile_line(req) else { continue; - } + }; let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( @@ -2898,7 +2952,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet"))] if is_not_pro { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(Error::from(anyhow::anyhow!("S3 pull was canceled"))), @@ -3430,7 +3484,10 @@ pub async fn start_worker( ) .await; - let mut proc_envs = HashMap::new(); + let mut proc_envs: HashMap = PYTHON_UTF8_ENVS + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); let additional_python_paths_folders = additional_python_paths.iter().join(":"); proc_envs.insert("PYTHONPATH".to_string(), additional_python_paths_folders); proc_envs.insert("PATH".to_string(), PATH_ENV.to_string()); @@ -3722,6 +3779,44 @@ mod tests { .is_ok()); } + #[tokio::test] + async fn test_verify_wheel_record_accepts_csv_quoted_path() { + let dir = tempfile::tempdir().unwrap(); + // A path containing a comma is CSV-quoted in RECORD (wcwidth 0.8.3 + // ships `wcwidth/textwrap.py,cover`). Splitting on the first comma + // looked for `"pkg/textwrap.py` and rejected a complete install. + write_fake_wheel( + dir.path(), + &["pkg/textwrap.py", "pkg/textwrap.py,cover"], + &[ + "pkg/textwrap.py,sha256=aaa,1", + "\"pkg/textwrap.py,cover\",sha256=bbb,1", + "pkg-1.0.0.dist-info/RECORD,,", + ], + ); + assert!(verify_wheel_record(dir.path().to_str().unwrap()) + .await + .is_ok()); + } + + #[test] + fn test_record_first_field_unquotes_rfc4180() { + assert_eq!( + record_first_field("pkg/a.py,sha256=x,1").as_deref(), + Some("pkg/a.py") + ); + assert_eq!( + record_first_field("\"pkg/a.py,cover\",sha256=x,1").as_deref(), + Some("pkg/a.py,cover") + ); + assert_eq!( + record_first_field("\"pkg/say \"\"hi\"\".py\",sha256=x,1").as_deref(), + Some("pkg/say \"hi\".py") + ); + assert_eq!(record_first_field(",,"), None); + assert_eq!(record_first_field("\"unterminated,sha256=x,1"), None); + } + // Regression tests for the concurrent-install guard. Two jobs installing the // same uncached dep into the shared `venv_p` used to race uv's `--reinstall`, // corrupting the on-disk wheel and failing with "Env installation did not diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 647d569cbe..0490ed1754 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -16,10 +16,6 @@ use windmill_common::otel_oss::FutureExt; use uuid::Uuid; -/// Set by the result processor when a WAC child completion makes suspend reach 0, -/// signaling the worker main loop to check for suspended jobs immediately. -pub static WAC_SUSPEND_READY: AtomicBool = AtomicBool::new(false); - use windmill_common::{ add_time, error::{self, Error}, @@ -36,8 +32,8 @@ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, - CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, + parse_result_object, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -76,13 +72,11 @@ struct NestedErrorMessage { /// named `name`/`message`), and we want OTel to record the ManualFailure /// rather than the user's sibling fields. fn extract_error_message(raw: &str) -> Option { - let nested = serde_json::from_str::(raw) - .ok() - .map(|n| n.error); + let nested = parse_result_object::(raw).map(|n| n.error); if matches!(&nested, Some(em) if em.name == MANUAL_FAILURE_ERROR_NAME) { return nested; } - if let Ok(em) = serde_json::from_str::(raw) { + if let Some(em) = parse_result_object::(raw) { return Some(em); } nested @@ -816,8 +810,19 @@ pub async fn handle_receive_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] #[derive(serde::Deserialize)] struct GitSyncCheck { - check_run_id: i64, - repo_url: String, + /// Absent when the repository's host has no check surface (GitLab): the + /// result then reaches the pull request through the managed comment alone. + #[serde(default)] + check_run_id: Option, + /// Only markers written before the repository URL moved out of job args + /// carry one; the resource path on the job is what is used now. + #[serde(default)] + repo_url: Option, + /// Host and path of the repository the check was created on, with no + /// credential in it. The resource path is mutable, so this is what proves + /// the resource still points where the check lives. + #[serde(default)] + repo: Option, #[serde(default)] pr_number: Option, #[serde(default)] @@ -972,19 +977,21 @@ mod git_sync_check_tests { } } -/// When an auto-pull job (carrying `__git_sync_auto_pull`) fails, roll the +/// When an auto-pull job (carrying `__git_sync_auto_pull`) completes: on success, +/// record the commit as a head the workspace reflects; on failure, roll the /// optimistic `last_synced_sha` advance back to the pre-pull value so the commit /// is retried instead of being silently treated as synced, and record the failure. +/// The recorded commit is the one the pull script reports having checked out +/// (`{sha, branch}` in its result): the branch can move between the observation +/// the marker holds and the clone. A result without it falls back to the marker. #[cfg(all(feature = "enterprise", feature = "private"))] async fn maybe_reconcile_git_sync_auto_pull( db: &DB, job_id: &uuid::Uuid, workspace_id: &str, success: bool, + result: &str, ) { - if success { - return; // the optimistic synced state is already correct - } let marker: Option = match sqlx::query_scalar!( "SELECT args->'__git_sync_auto_pull' FROM v2_job WHERE id = $1", job_id @@ -1004,12 +1011,50 @@ async fn maybe_reconcile_git_sync_auto_pull( #[derive(serde::Deserialize)] struct AutoPullMarker { repo_resource_path: String, + branch: Option, + head_sha: Option, #[serde(default)] prev_synced: std::collections::HashMap, } let Ok(m) = serde_json::from_value::(marker) else { return; }; + if success { + // The optimistic synced state is already correct; record that the workspace + // now reflects the commit, which the PR CI-test check waits for. + #[derive(serde::Deserialize)] + struct PullResult { + sha: Option, + branch: Option, + } + let applied = serde_json::from_str::(result).ok(); + let branch = applied + .as_ref() + .and_then(|r| r.branch.as_deref()) + .or(m.branch.as_deref()); + let sha = applied + .as_ref() + .and_then(|r| r.sha.as_deref()) + .or(m.head_sha.as_deref()); + if let (Some(branch), Some(sha)) = (branch, sha) { + if let Err(e) = windmill_git_sync::record_synced_head( + db, + workspace_id, + &m.repo_resource_path, + branch, + sha, + "pull", + Some(*job_id), + ) + .await + { + tracing::warn!( + "git auto-pull: failed to record synced head {sha} on {branch}: {e:#}" + ); + } + } + return; + } windmill_git_sync::record_auto_pull_failure( db, workspace_id, @@ -1098,6 +1143,70 @@ fn git_sync_push_result_pushed(result: &str) -> Option { .as_bool() } +/// When a git-sync push job pushed a commit, record it as a head the workspace +/// reflects, the way a successful pull records the commit it applied. The PR +/// CI-test check waits for that record. Best-effort: failures are logged, never +/// propagated. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maybe_record_git_sync_pushed_head( + db: &DB, + job_id: &uuid::Uuid, + workspace_id: &str, + result: &str, +) { + #[derive(serde::Deserialize)] + struct PushResult { + pushed: bool, + sha: Option, + branch: Option, + #[serde(default)] + rebased: bool, + } + let Ok(PushResult { pushed: true, sha: Some(sha), branch: Some(branch), rebased }) = + serde_json::from_str::(result) + else { + return; + }; + // A push that had to rebase sits on commits this workspace has not pulled, so the + // pushed head is not something it reflects yet; the pull those commits trigger + // records the head once they are in. + if rebased { + tracing::info!( + "git sync push: {sha} on {branch} was rebased onto unpulled commits; not recording it as synced for {workspace_id}" + ); + return; + } + let repo_path = match sqlx::query_scalar!( + "SELECT args->>'repo_url_resource_path' FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(db) + .await + { + Ok(Some(Some(p))) => p, + Ok(_) => return, + Err(e) => { + tracing::error!("git sync push: failed to read job args: {e:#}"); + return; + } + }; + if let Err(e) = windmill_git_sync::record_synced_head( + db, + workspace_id, + &repo_path, + &branch, + &sha, + "push", + Some(*job_id), + ) + .await + { + tracing::warn!( + "git sync push: failed to record pushed head {sha} on {branch} for {workspace_id}/{repo_path}: {e:#}" + ); + } +} + /// When a git-sync push job carrying `__git_sync_open_pr` succeeds, open (or /// reopen) the PR for the branch it pushed: `wm-fork//` for a fork /// deploy, `wm_deploy/**` for a promotion deploy. Runs outbound with the @@ -1145,35 +1254,23 @@ async fn maybe_open_git_sync_deploy_pr( if row.marker.is_none() { return; } - // Runtime Enterprise gate, like the poller: the toggles may have been set - // while a license was active (or written directly), and this hook drives - // GitHub API calls with the installation token. - if !matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ) { - tracing::warn!( - "git sync PR: skipping PR creation for {workspace_id}: requires an Enterprise license" - ); - return; - } let Some(repo_path) = row.repo_path else { return; }; // Base = the tracked branch (resource branch, else the repo default). Also - // acts as the app-backed gate: PR creation needs the installation token. - let base = match windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + // acts as the gate: PR creation needs a credential the server itself holds. + let base = match windmill_common::git_sync_ee::managed_pr_base_branch( db, workspace_id, &repo_path, ) .await { - Ok(Some((branch, _))) => branch, + Ok(Some(branch)) => branch, Ok(None) => { tracing::warn!( - "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but is not GitHub-App-backed; skipping (connect the repo through the GitHub App, or use the open-pr-on-commit workflow)" + "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but the server holds no credential for it; skipping (connect the repo through the GitHub App or a GitLab token, or use the open-pr-on-commit workflow)" ); return; } @@ -1373,22 +1470,59 @@ async fn maybe_post_git_sync_check( (None, Some(deploy)) => (true, deploy), (None, None) => return, }; - let Ok(mut check) = serde_json::from_value::(marker) else { + let Ok(check) = serde_json::from_value::(marker) else { return; }; - // Markers carry the literal resource URL (job args are persisted, so a - // `$var:`-resolved URL must not land there); interpolate before calling - // GitHub. - check.repo_url = - match windmill_common::variables::get_variable_or_self(check.repo_url, db, workspace_id) - .await - { - Ok(u) => u, - Err(e) => { - tracing::error!("git sync-check: cannot interpolate repo url: {e:#}"); - return; - } - }; + // Job args are persisted, so the repository URL is not among them: it is + // re-resolved here from the resource path the pull job carries. A marker + // written before that change still has the URL, and is honoured until the + // last such job has drained. + // The resource path is mutable, so it is only trusted when the marker also + // carries the identity to check it against. A marker written before that + // identity existed keeps using the URL it captured at enqueue, which cannot + // have been repointed since. + let repo_url = match ( + check.repo.is_some(), + row.repo_path.as_deref(), + check.repo_url.clone(), + ) { + // The resource path is mutable, so following it is only safe when the + // marker also carries the identity to check the result against. + (true, Some(path), _) => { + windmill_common::git_sync_ee::resolve_repo_url_interpolated(db, workspace_id, path) + .await + } + // A marker written before that identity existed captured the URL itself, + // which cannot have been repointed since. + (_, _, Some(url)) => { + windmill_common::variables::get_variable_or_self(url, db, workspace_id).await + } + // Neither: nothing here can prove which repository this check belongs to, + // and resolving the path anyway is how a preview reaches the wrong one. + // Leaving the check unfinished is the safe failure. + _ => { + tracing::error!( + "git sync-check: the marker carries neither a repository identity nor a url; not acting on it" + ); + return; + } + }; + let repo_url = match repo_url { + Ok(u) => u, + Err(e) => { + tracing::error!("git sync-check: cannot resolve repo url: {e:#}"); + return; + } + }; + // A resource repointed while the diff was running would otherwise close a + // check, or post a preview, on a repository that has nothing to do with it. + if check.repo.is_some() && windmill_common::git_sync_ee::repo_identity(&repo_url) != check.repo + { + tracing::warn!( + "git sync-check: the repository moved since the check was created; leaving it alone" + ); + return; + } // "In sync" on a PR that visibly changes files reads as a bug when those // files are outside the repo's sync filters — say what the scope is. let scope_note = if !is_deploy && success { @@ -1451,7 +1585,7 @@ async fn maybe_post_git_sync_check( ( "failure", "Merge conflicts with the base branch".to_string(), - "This PR cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." + "This branch cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." .to_string(), ) } else if pr_check_error.as_deref() == Some("PR_HEAD_REF_UNAVAILABLE") { @@ -1461,7 +1595,7 @@ async fn maybe_post_git_sync_check( ( "neutral", "Could not compute the deploy diff".to_string(), - "Windmill could not fetch this PR's head or enough history from GitHub to compute its merge with the base. Push again to re-run this check." + "Windmill could not fetch this branch's head, or enough history, to compute its merge with the base. Push again to re-run this check." .to_string(), ) } else if pr_check_error.is_some() { @@ -1484,20 +1618,20 @@ async fn maybe_post_git_sync_check( "success", "In sync".to_string(), format!( - "Merging this PR would make no changes to the workspace.{}", + "Merging this branch would make no changes to the workspace.{}", scope_note.as_deref().unwrap_or_default() ), ), Some((changes, settings_changed)) => { let mut lines = vec![format!( - "Merging this PR would apply {} change(s) to the workspace:\n", + "Merging this branch would apply {} change(s) to the workspace:\n", changes.len() )]; lines.extend(format_change_list(&changes)); if settings_changed { lines.push(match check.wmill_yaml_changed { - Some(true) => "\nThis PR changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), - Some(false) => "\nIndependent of this PR, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), + Some(true) => "\nThis branch changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), + Some(false) => "\nIndependent of this branch, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), None => "\nA pull also updates the workspace's git-sync settings to match the repo's wmill.yaml.".to_string(), }); } @@ -1520,19 +1654,21 @@ async fn maybe_post_git_sync_check( Some(url) => format!("{summary}\n\n[See the job in Windmill]({url})"), None => summary.clone(), }; - if let Err(e) = windmill_common::git_sync_ee::update_check_run( - db, - workspace_id, - &check.repo_url, - check.check_run_id, - conclusion, - &title, - &check_summary, - job_url.as_deref(), - ) - .await - { - tracing::error!("git sync-check: failed to update check run: {e:#}"); + if let Some(check_run_id) = check.check_run_id { + if let Err(e) = windmill_common::git_sync_ee::update_check_run( + db, + workspace_id, + &repo_url, + check_run_id, + conclusion, + &title, + &check_summary, + job_url.as_deref(), + ) + .await + { + tracing::error!("git sync-check: failed to update check run: {e:#}"); + } } // Phase 4 also maintains ONE managed comment on the PR (Cloudflare @@ -1556,7 +1692,7 @@ async fn maybe_post_git_sync_check( if let Err(e) = windmill_common::git_sync_ee::upsert_pr_comment( db, workspace_id, - &check.repo_url, + &repo_url, pr_number, marker, &body, @@ -1640,7 +1776,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration, wac_job_ids) = add_completed_job( + let (_, duration) = add_completed_job( db, &job, true, @@ -1657,8 +1793,26 @@ pub async fn process_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { maybe_post_git_sync_check(db, &job_id, &workspace_id, true, result.get()).await; + maybe_reconcile_git_sync_auto_pull(db, &job_id, &workspace_id, true, result.get()) + .await; + maybe_record_git_sync_pushed_head(db, &job_id, &workspace_id, result.get()).await; maybe_open_git_sync_deploy_pr(db, &job_id, &workspace_id, result.get()).await; } + // A CI test job just finished: advance any open "Windmill CI tests" PR check for + // its workspace. Detached, since concluding a check calls GitHub and this loop + // completes jobs serially; the evaluation is idempotent and the poller retries. + #[cfg(all(feature = "enterprise", feature = "private"))] + if job + .trigger_kind + .as_ref() + .is_some_and(|k| k.is(windmill_common::jobs::JobTriggerKind::CiTest)) + { + let db = db.clone(); + let w_id = workspace_id.clone(); + tokio::spawn(async move { + windmill_git_sync::evaluate_and_conclude_ci_test_checks(&db, &w_id).await + }); + } // Asset-trigger fan-out: best-effort, never propagates errors. // Internal eligibility checks gate to top-level Script/Preview runs; @@ -1703,29 +1857,6 @@ pub async fn process_completed_job( } return Ok(r); } - } else if let Some(parent_job) = parent_job { - // wac_job_ids is piggybacked from the duration write in - // add_completed_job — no extra query needed. - if let Some(job_ids) = wac_job_ids { - if let Ok(Some(_)) = handle_wac_child_completion( - db, - &job_id, - parent_job, - &workspace_id, - result, - true, - job_ids, - ) - .await - { - if let Some(done_tx) = done_tx { - done_tx - .send(()) - .expect("done receiver should still be alive"); - } - return Ok(None); - } - } } } else { // The result already carries our injected @@ -1769,7 +1900,20 @@ pub async fn process_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { maybe_post_git_sync_check(db, &job.id, &job.workspace_id, false, result.get()).await; - maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false).await; + maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false, "").await; + } + // A failed CI test job also settles its check; same detached advance as on success. + #[cfg(all(feature = "enterprise", feature = "private"))] + if job + .trigger_kind + .as_ref() + .is_some_and(|k| k.is(windmill_common::jobs::JobTriggerKind::CiTest)) + { + let db = db.clone(); + let w_id = job.workspace_id.clone(); + tokio::spawn(async move { + windmill_git_sync::evaluate_and_conclude_ci_test_checks(&db, &w_id).await + }); } if job.is_flow_step() { if let Some(parent_job) = job.parent_job { @@ -1809,227 +1953,11 @@ pub async fn process_completed_job( } return Ok(r); } - } else if let Some(parent_job) = job.parent_job { - // WAC child failed — query job_ids from parent (errors are rare, - // so the extra read is acceptable here). - let job_ids_json: Option> = sqlx::query_scalar( - "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ - FROM v2_job_status WHERE id = $1", - ) - .bind(&parent_job) - .fetch_optional(db) - .await?; - if let Some(Some(job_ids)) = job_ids_json { - if let Ok(Some(_)) = handle_wac_child_completion( - db, - &job.id, - parent_job, - &job.workspace_id, - downstream_result, - false, - job_ids, - ) - .await - { - if let Some(done_tx) = done_tx { - done_tx - .send(()) - .expect("done receiver should still be alive"); - } - return Ok(None); - } - } } } return Ok(None); } -/// Handle a WAC v2 child job completion. -/// Returns Ok(Some(())) if the parent was a WAC job and was handled, -/// Ok(None) if the parent is not a WAC job (caller should fall through). -/// -/// CONCURRENCY: Multiple parallel children may complete simultaneously on -/// different workers. We use atomic SQL operations throughout: -/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))` -/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each -/// worker sees the previous worker's writes. -/// - The suspend counter (set to N at dispatch time) is decremented atomically -/// with `RETURNING` to determine the "all done" condition. -pub(crate) async fn handle_wac_child_completion( - db: &DB, - child_job_id: &Uuid, - parent_job_id: Uuid, - workspace_id: &str, - result: Arc>, - success: bool, - job_ids_value: Value, -) -> error::Result> { - let job_ids = match job_ids_value { - Value::Object(m) => m, - _ => return Ok(None), // Not a WAC parent or no pending steps - }; - - let child_id_str = child_job_id.to_string(); - let step_key = job_ids.iter().find_map(|(key, val)| { - if val.as_str() == Some(&child_id_str) { - Some(key.clone()) - } else { - None - } - }); - - let step_key = match step_key { - Some(k) => k, - None => { - if !success { - // No step key and failed — can't store error, fail parent immediately - tracing::error!( - parent_job = %parent_job_id, - child_job = %child_job_id, - "WAC v2 child job failed but no step key found, failing parent" - ); - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", - parent_job_id, - ) - .execute(db) - .await?; - let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?; - if let Some(parent_mini) = parent_mini { - let child_err: Value = - serde_json::from_str(result.get()).unwrap_or(Value::Null); - let err_value = json!({ - "message": format!("WAC child job {} failed (no step key)", child_job_id), - "error": child_err, - }); - let _ = windmill_queue::add_completed_job_error( - db, - &parent_mini, - 0, - None, - err_value, - "wac_child_handler", - false, - None, - ) - .await; - } - return Ok(Some(())); - } - tracing::warn!( - parent_job = %parent_job_id, - child_job = %child_job_id, - "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang" - ); - // Still decrement suspend so the parent doesn't hang indefinitely - let _ = sqlx::query_scalar!( - "UPDATE v2_job_queue \ - SET suspend = GREATEST(suspend - 1, 0) \ - WHERE id = $1 \ - RETURNING suspend", - parent_job_id, - ) - .fetch_optional(db) - .await?; - return Ok(Some(())); - } - }; - - // Build result — wrap errors with _error marker so workflow try/catch can handle them - let result_value: Value = if success { - serde_json::from_str(result.get()).unwrap_or(Value::Null) - } else { - let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null); - tracing::info!( - parent_job = %parent_job_id, - child_job = %child_job_id, - step_key = %step_key, - "WAC v2 child job failed, storing error for workflow try/catch" - ); - windmill_common::wac::wac_failure_record( - &step_key, - Some(&child_job_id.to_string()), - &child_err, - ) - }; - - tracing::info!( - parent_job = %parent_job_id, - child_job = %child_job_id, - step_key = %step_key, - success = success, - "WAC v2 child job completed" - ); - - // Use a transaction to ensure completed_steps merge + suspend decrement - // are atomic. Without this, a crash between the two could strand the parent. - let result_json = serde_json::to_value(&result_value) - .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?; - - let mut tx = db.begin().await?; - - // Merge the completed step into the checkpoint. - // Uses `|| jsonb_build_object(key, value)` so concurrent children on - // different workers don't overwrite each other — PostgreSQL serialises - // concurrent UPDATEs on the same row and each sees the previous write. - sqlx::query( - "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( - workflow_as_code_status, - '{_checkpoint,completed_steps}', - COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) - || jsonb_build_object($2::text, $3::jsonb) - ) WHERE id = $1", - ) - .bind(&parent_job_id) - .bind(&step_key) - .bind(&result_json) - .execute(&mut *tx) - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?; - - // Decrement the suspend counter. The counter was set to N (number of - // children) at dispatch time. When it reaches 0 all children are done. - // Keep suspend_until non-null so the suspended pull query - // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent. - let new_suspend: Option = sqlx::query_scalar!( - "UPDATE v2_job_queue \ - SET suspend = GREATEST(suspend - 1, 0) \ - WHERE id = $1 \ - RETURNING suspend", - parent_job_id, - ) - .fetch_optional(&mut *tx) - .await?; - - let all_done = new_suspend == Some(0); - - if all_done { - // Clear pending_steps from checkpoint since all children are complete. - // This is cosmetic — the next replay will overwrite it anyway — but - // keeps the checkpoint clean for frontend display. - let _ = sqlx::query( - "UPDATE v2_job_status SET workflow_as_code_status = \ - workflow_as_code_status #- '{_checkpoint,pending_steps}' \ - WHERE id = $1", - ) - .bind(&parent_job_id) - .execute(&mut *tx) - .await; - } - - tx.commit().await?; - - if all_done { - tracing::info!( - parent_job = %parent_job_id, - "WAC v2 all child jobs completed, unsuspending parent" - ); - WAC_SUSPEND_READY.store(true, Ordering::Relaxed); - } - - Ok(Some(())) -} - pub async fn handle_non_flow_job_error( db: &DB, job: &MiniCompletedJob, diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index b9d53a801b..dff7d58c2d 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -14,7 +14,6 @@ use tokio::{ }; use windmill_common::{ error::{self, Error}, - utils::calculate_hash, worker::{write_file, Connection}, }; use windmill_queue::MiniPulledJob; @@ -602,8 +601,13 @@ pub async fn build_rust_crate( /// Cache key of a Rust build. The run path and the deploy-time prebuild must derive it /// the same way or the prebuilt binary is never found and gets rebuilt on first run. -async fn rust_cache_key(code: &str, requirements_o: Option<&String>, w_id: &str) -> String { - let mut hash = compute_rust_hash(code, requirements_o); +async fn rust_cache_key( + code: &str, + requirements_o: Option<&String>, + w_id: &str, + modules: Option<&HashMap>, +) -> String { + let mut hash = compute_rust_hash(code, requirements_o, modules); hash.push_str(&crate::workspace_registry_cache_suffix(w_id).await); hash } @@ -621,11 +625,12 @@ pub async fn prebuild_rust_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> error::Result> { ensure_rust_runtime_dirs(); check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; - let hash = rust_cache_key(code, Some(&lock.to_string()), &job.workspace_id).await; + let hash = rust_cache_key(code, Some(&lock.to_string()), &job.workspace_id, modules).await; let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -652,15 +657,22 @@ pub async fn prebuild_rust_binary( Ok(Some(logs)) } -pub fn compute_rust_hash(code: &str, requirements_o: Option<&String>) -> String { - calculate_hash(&format!( +pub fn compute_rust_hash( + code: &str, + requirements_o: Option<&String>, + // Companion modules are written into the crate dir and compiled into the binary this + // key names, so leaving them out shares one script's binary with another. + modules: Option<&HashMap>, +) -> String { + let base = format!( "{}{}", code, requirements_o .as_ref() .map(|x| x.to_string()) .unwrap_or_default() - )) + ); + crate::worker::artifact_cache_name(base, modules) } #[tracing::instrument(level = "trace", skip_all)] @@ -679,11 +691,12 @@ pub async fn handle_rust_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { ensure_rust_runtime_dirs(); check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; - let hash = rust_cache_key(inner_content, requirements_o, &job.workspace_id).await; + let hash = rust_cache_key(inner_content, requirements_o, &job.workspace_id, modules).await; let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index f36a5e6fc9..1126c94d17 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -333,7 +333,7 @@ pub async fn par_install_language_dependencies_all_at_once< mark_success(path.clone(), job_id, w_id).await; #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let language_name = _language_name.to_owned(); tokio::spawn(async move { if let Err(e) = crate::global_cache::build_tar_and_push( @@ -790,7 +790,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] let s3_pull_future = if is_not_pro { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { Some(crate::global_cache::pull_from_tar( os, dep.path.clone(), @@ -893,7 +893,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + #[cfg(all(feature = "enterprise", feature = "parquet"))] { - if let Some(os) = windmill_object_store::get_object_store().await { + if let Some(os) = windmill_object_store::get_cache_object_store().await { let language_name = _language_name.to_string(); let platform_agnostic = _platform_agnostic; let path = dep.path.clone(); @@ -954,8 +954,7 @@ async fn print_success( } #[cfg(all(feature = "enterprise", feature = "parquet"))] - if windmill_object_store::OBJECT_STORE_SETTINGS - .read() + if windmill_object_store::get_cache_object_store() .await .is_none() { diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 3e9b7bbc9b..c188792a5e 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -1,11 +1,13 @@ use serde::Deserialize; use serde_json::value::RawValue; use serde_json::Value; +use sqlx::{Postgres, Transaction}; use uuid::Uuid; use windmill_common::error::{self, Error}; use windmill_common::scripts::ScriptLang; use windmill_common::DB; +use windmill_queue::CanceledBy; // Checkpoint model + persistence primitives live in windmill-common so the // API server can use them without pulling in the full worker crate. Re-export @@ -44,6 +46,10 @@ pub enum WacOutput { form: Option, #[serde(default)] self_approval_disabled: Option, + #[serde(default)] + skin: Option, + #[serde(default)] + description: Option, }, /// Server-side sleep — suspend the workflow for a duration without holding a worker. #[serde(rename = "sleep")] @@ -85,6 +91,127 @@ fn default_dispatch_type() -> String { "inline".to_string() } +/// What `suspend_wac_parent` did with the parent's queue row. +#[derive(Debug)] +pub enum WacPark { + /// Parked. Carries the segment that just ended, in milliseconds, for `end_wac_segment`. + Parked(Option), + /// A cancel reached the row while this segment was running, so the park was skipped. + /// Carries who cancelled, for the completion that must happen instead. + Cancelled(CanceledBy), +} + +/// Park a WAC v2 parent in the queue until `suspend` reaches 0 or `suspend_secs` +/// elapses, whichever comes first. `running` stays true so the normal pull query +/// skips the row; only the suspended pull query takes it back. The `id`/`workspace_id` +/// pair is a consistency check, not an authorization one — callers must already hold +/// the job (every one of them passes a job its own worker pulled). +/// +/// `started_at` is cleared because the parent holds no worker while parked. The pull +/// re-stamps it (`started_at = coalesce(started_at, now())`), and every path that +/// completes a job without a worker-measured duration — a cancel, the child-failure +/// handler — falls back to `now() - started_at`. Left pointing at the first segment, +/// that fallback reports the whole sleep or approval wait as execution time. +/// +/// Call it before any write to the parent's `v2_job_status` row in the same +/// transaction: a child's completion locks the queue row and then the status row +/// (`record_child_completion`), and taking them the other way round here can +/// deadlock against a stale child finishing while the parent re-dispatches. +pub async fn suspend_wac_parent( + tx: &mut Transaction<'_, Postgres>, + job_id: &Uuid, + w_id: &str, + suspend: i32, + suspend_secs: f64, +) -> error::Result { + // `FOR UPDATE` orders this against a concurrent soft cancel, which writes `suspend = 0` + // and leaves acting on `canceled_by` to the next pull. Parking on top of that keeps the + // row unpullable until `suspend_until` — up to the full `sleep()` — so a cancel already + // on the row has to stand the park down rather than be overwritten by it. + let prev = sqlx::query!( + "SELECT canceled_by, canceled_reason, + (extract(epoch FROM now() - started_at) * 1000)::bigint AS segment_ms + FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 FOR UPDATE", + job_id, + w_id, + ) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to read WAC parent job {job_id}: {e}")))? + // Silently parking nothing is unrecoverable on the dispatch arm: the children are + // pushed right after and decrement a `suspend` that was never set, so the parent + // sits out its whole suspend window instead of resuming. + .ok_or_else(|| { + Error::internal_err(format!( + "WAC parent job {job_id} not in the queue of workspace {w_id} to suspend" + )) + })?; + + if let Some(username) = prev.canceled_by { + return Ok(WacPark::Cancelled(CanceledBy { + username: Some(username), + reason: prev.canceled_reason, + })); + } + + sqlx::query!( + "UPDATE v2_job_queue + SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null + WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + suspend, + suspend_secs, + ) + .execute(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to suspend WAC parent job {job_id}: {e}")))?; + + Ok(WacPark::Parked(prev.segment_ms)) +} + +/// Turn a cancel that landed mid-segment into the error the executor returns, so the job +/// completes on this pass instead of parking. Setting the worker's `canceled_by` is what +/// makes it land as `canceled` rather than `failure`: the row was cancelled after this +/// worker pulled the job, so the in-memory copy still reads as uncancelled. +/// +/// The completion charges the segment that just ended, so callers must not also hand it to +/// `end_wac_segment`. +pub(crate) fn wac_cancelled_mid_segment( + cancel: CanceledBy, + canceled_by: &mut Option, +) -> Error { + let payload = windmill_common::worker::to_raw_value(&windmill_queue::canceled_result( + cancel.reason.as_deref(), + cancel.username.as_deref(), + )); + *canceled_by = Some(cancel); + Error::ExecutionRawError(payload) +} + +/// Charge the execution segment a WAC parent just finished. Segments are metered as they +/// end rather than summed at completion, so a workflow that sleeps for days is billed for +/// the compute it used, when it used it — and the final segment is charged by the ordinary +/// completion path. +/// +/// Call this only where the parent really parks. On a rollback that goes on to complete +/// the job, the completion charges the same segment and it would be billed twice. +pub(crate) fn end_wac_segment( + _conn: &windmill_common::worker::Connection, + _job: &windmill_queue::MiniPulledJob, + _segment_ms: Option, +) { + #[cfg(feature = "cloud")] + if let (windmill_common::worker::Connection::Sql(db), Some(segment_ms)) = (_conn, _segment_ms) { + windmill_queue::meter_execution_seconds( + db, + &_job.workspace_id, + &_job.permissioned_as_email, + segment_ms, + ); + } +} + /// Parse the WAC result from result.json content. pub fn parse_wac_output(result: &RawValue) -> error::Result { serde_json::from_str(result.get()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 350f0c5a61..55a698a9dd 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -950,6 +950,51 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String { } } +/// The name a build artifact is cached under, derived from `base` — the runnable's own +/// cache-key input — and its inline modules. +/// +/// `write_module_files` puts module content in the job dir where the build inlines it into +/// the artifact, so a name without it serves one runnable's modules to another whose main +/// content and lockfile match — across workspaces, the cache being global. +/// +/// Only the path and content may name the artifact, because they are all the build reads. +/// `ScriptModule::lock` especially must stay out: deploy regenerates it *after* the parent +/// has prebuilt, so naming it would strand every prebuilt artifact. +pub(crate) fn artifact_cache_name( + base: String, + modules: Option<&std::collections::HashMap>, +) -> String { + let Some(modules) = modules.filter(|m| !m.is_empty()) else { + // Byte-identical to the name a module-free runnable had before modules entered this, + // so its cached artifacts stay reachable. A pre-fix multi-file runnable also stored + // here, so one of those stays reachable too — accepted over invalidating every cache, + // and once this ships nothing can be stored here with module content in it again. + return windmill_common::utils::calculate_hash(&base); + }; + let mut entries: Vec<(&String, &ScriptModule)> = modules.iter().collect(); + entries.sort_by(|a, b| a.0.cmp(b.0)); + // `base` ends in caller-supplied bytes (a preview brings its own lockfile), so it is + // sealed to a fixed width before the module block is appended — raw, a crafted lockfile + // could spell out another runnable's block and reach its slot. + let mut keyed = format!( + "{}:modules:{}", + windmill_common::utils::calculate_hash(&base), + entries.len() + ); + for (path, module) in entries { + // Both length-prefixed, else `{"a": "bc"}` and `{"ab": "c"}` encode alike. + keyed.push_str(&format!( + ":{}:{path}:{}:{}", + path.len(), + module.content.len(), + module.content, + )); + } + // Its own namespace: `calculate_hash` emits hex, so however a module-free runnable + // crafts its content and lockfile it can never land on a module-bearing name. + format!("mod-{}", windmill_common::utils::calculate_hash(&keyed)) +} + pub fn is_sandboxing_enabled() -> bool { if !*DISABLE_NSJAIL { return true; @@ -3360,7 +3405,7 @@ pub async fn run_worker( let suspend_first = suspend_first_success || rand::random::() < likelihood_of_suspend || last_suspend_first.elapsed().as_secs_f64() > 5.0 - || crate::result_processor::WAC_SUSPEND_READY + || windmill_common::wac::WAC_SUSPEND_READY .swap(false, Ordering::Relaxed); if suspend_first { @@ -3532,7 +3577,13 @@ pub async fn run_worker( job.kind, JobKind::Script | JobKind::Preview | JobKind::FlowScript ) { - if !dedicated_workers.is_empty() { + // A job carrying a pre-run error never runs its code: it only has to be + // pulled so `handle_queued_job` can fail it. Both hand-off paths below + // dispatch by path and return before that check, so a job sent down them + // would run with whatever arguments survived the failure. + let fails_before_running = job.pre_run_error.is_some(); + + if !dedicated_workers.is_empty() && !fails_before_running { let dedicated_worker_tx = job.runnable_path.as_ref().and_then(|path| { // For flow steps inside branches/loops, runnable_path includes // nesting segments (e.g. f/flow/branchone-0/a) but the dedicated @@ -3577,7 +3628,7 @@ pub async fn run_worker( NextJob::Http(_) => None, }; - if let Some(flow_runners) = flow_runners { + if let Some(flow_runners) = flow_runners.filter(|_| !fails_before_running) { let key_o = job.flow_step_id.as_ref().map(|x| x.to_string()); if let Some(key) = key_o { if let Some(flow_runner_tx) = flow_runners.runners.get(&key) { @@ -3808,15 +3859,14 @@ pub async fn run_worker( let arc_job = Arc::new(job); - windmill_common::sensitive_log_masks::register_running_job(arc_job.id); - let span = create_span_with_name(&arc_job, &worker_name, Some(hostname), "job"); let log_ctx = log_context_for_job(&arc_job, &worker_name, Some(hostname)); let job_result = windmill_common::log_context::with_log_context( log_ctx, async { - let result = handle_queued_job( + // Keep large job-phase futures boxed to limit debug polling frames. + let result = Box::pin(handle_queued_job( arc_job.clone(), raw_code, raw_lock, @@ -3837,7 +3887,7 @@ pub async fn run_worker( flow_runners, #[cfg(feature = "benchmark")] &mut bench, - ) + )) .await; record_job_span_status(&result); result @@ -3924,8 +3974,6 @@ pub async fn run_worker( _ => {} } - windmill_common::sensitive_log_masks::unregister_running_job(job_id); - #[cfg(feature = "prometheus")] if let Some(duration) = _timer.map(|x| x.stop_and_record()) { register_metric( @@ -4460,6 +4508,30 @@ async fn detect_and_store_runtime_assets_from_job_args( } } +/// Holds a job's entry in the log-masking registry for as long as it executes, so +/// that secrets it fetches can be registered against it, and masks the job's own +/// token from the start: `$WM_TOKEN` stays valid well past the run, and a script +/// that echoes it would otherwise leave a live credential in the persisted logs. +/// +/// Lives here rather than at the call sites so that every way of running a job — +/// the poller, the interactive worker shell, an inline AI agent tool — is covered +/// by construction. +struct RunningJobMasks(Uuid); + +impl RunningJobMasks { + fn register(job_id: Uuid, token: &str) -> Self { + windmill_common::sensitive_log_masks::register_running_job(job_id); + windmill_common::sensitive_log_masks::register_secret_for_job(job_id, token); + RunningJobMasks(job_id) + } +} + +impl Drop for RunningJobMasks { + fn drop(&mut self) { + windmill_common::sensitive_log_masks::unregister_running_job(self.0); + } +} + pub async fn handle_queued_job( job: Arc, raw_code: Option, @@ -4481,6 +4553,8 @@ pub async fn handle_queued_job( flow_runners: Option>, #[cfg(feature = "benchmark")] _bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { + let _masks = RunningJobMasks::register(job.id, &client.token); + if job.canceled_by.is_some() { return Err(Error::JsonErr(canceled_job_to_result(&job))); } @@ -4593,7 +4667,7 @@ pub async fn handle_queued_job( let cached_res_path = if job.cache_ttl.is_some() { match conn { Connection::Sql(db) => { - Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await) + Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await?) } Connection::Http(_) => None, } @@ -5108,7 +5182,7 @@ async fn try_validate_schema( code, language, job.script_entrypoint_override.clone(), - )? { + ).await? { Ok(Some(schema_validator_from_main_arg_sig(&sig))) } else { Err(anyhow!("Job was expected to validate the arguments schema, but no schema was provided and couldn't be inferred from the script for language `{language:?}`. Try removing schema validation for this job").into()) @@ -5453,7 +5527,9 @@ async fn handle_code_execution_job( None => job, }; - // For preview jobs, extract modules from args._MODULES if not already set + // Any job kind, not just previews: whatever is here is what gets written to the job dir + // and built in, so the agent-worker server precomputing a cache name has to resolve + // modules the same way (`windmill-api-agent-workers`, `get_code_and_lock`). let modules = modules_from_data.clone().or_else(|| { job.args.as_ref().and_then(|args| { args.get("_MODULES").and_then(|raw| { @@ -5474,7 +5550,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = Box::pin(run_language_executor( job, conn, client, @@ -5499,8 +5575,111 @@ async fn handle_code_execution_job( &modules, false, in_pipeline, - ) - .await + )) + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot @@ -5646,11 +5825,106 @@ mod write_module_files_tests { use super::*; use std::collections::HashMap; use windmill_common::scripts::ScriptLang; + use windmill_common::utils::calculate_hash; fn module(content: &str) -> ScriptModule { ScriptModule { content: content.to_string(), language: ScriptLang::Python3, lock: None } } + /// Every language's artifact cache name funnels module content through this, so an + /// ambiguous encoding puts two different runnables back on one name. + #[test] + fn artifact_name_cannot_be_re_cut_into_another_module_map() { + fn name(entries: &[(&str, &str)]) -> String { + let map: HashMap = entries + .iter() + .map(|(p, c)| (p.to_string(), module(c))) + .collect(); + artifact_cache_name("base".to_string(), Some(&map)) + } + + // Naive `path + content` concatenation renders both of these as "abc". + assert_ne!(name(&[("a", "bc")]), name(&[("ab", "c")])); + // Splitting one module into two must not read back as the joined one. + assert_ne!(name(&[("a", "b"), ("c", "d")]), name(&[("ac", "bd")])); + // Iteration order of the map must not move the name. + assert_eq!( + name(&[("a", "1"), ("b", "2")]), + name(&[("b", "2"), ("a", "1")]) + ); + } + + /// A module-free runnable must keep the exact name it had before modules entered the + /// derivation, or upgrading strands every artifact already in the cache. + #[test] + fn artifact_name_is_unchanged_without_modules() { + assert_eq!( + artifact_cache_name("code+lock".to_string(), None), + calculate_hash("code+lock") + ); + assert_eq!( + artifact_cache_name("code+lock".to_string(), Some(&HashMap::new())), + calculate_hash("code+lock") + ); + } + + /// A preview brings its own source and lockfile, so a module-free runnable picks its + /// whole `base`. Module-bearing names live in their own namespace precisely so that no + /// crafted `base` can be made to land on one. + #[test] + fn a_module_free_runnable_cannot_forge_a_module_bearing_name() { + let modules = HashMap::from([("h.ts".to_string(), module("evil"))]); + let victim = artifact_cache_name("code+lock".to_string(), Some(&modules)); + + // `calculate_hash` emits hex, so the namespace is unreachable however `base` is + // chosen — including by feeding it the victim's own name. + assert!(victim.starts_with("mod-")); + assert_ne!(artifact_cache_name(victim.clone(), None), victim); + assert!(!artifact_cache_name("anything".to_string(), None).starts_with("mod-")); + } + + /// The `mod-` namespace separates module-free from module-bearing, and nothing separates + /// two module-bearing runnables — only the seal does. Unsealed, `base` is variable-width, + /// so the split between it and the module block is ambiguous and a preview (which brings + /// its own source *and* lockfile) can absorb part of another runnable's block. + #[test] + fn a_module_bearing_runnable_cannot_absorb_another_ones_block() { + let victim = artifact_cache_name( + "V".to_string(), + Some(&HashMap::from([( + "h.ts".to_string(), + module(":modules:1:1:a:1:b"), + )])), + ); + // Byte-identical to the victim's without the seal: the forger's `base` spells out the + // victim's leading block, leaving its own single module to supply the tail. + let forged = artifact_cache_name( + "V:modules:1:4:h.ts:18:".to_string(), + Some(&HashMap::from([("a".to_string(), module("b"))])), + ); + + assert_ne!(victim, forged); + } + + /// Deploy fills a module's lock in after the parent has prebuilt, so a name that moved + /// with it would leave every prebuilt artifact unreachable by the runs it was built for. + #[test] + fn artifact_name_ignores_the_lock_deploy_fills_in_later() { + let prebuild = artifact_cache_name( + "base".to_string(), + Some(&HashMap::from([("h.ts".to_string(), module("x"))])), + ); + + let mut locked = module("x"); + locked.lock = Some("{}\n//bun.lock\n".to_string()); + let after_deploy = artifact_cache_name( + "base".to_string(), + Some(&HashMap::from([("h.ts".to_string(), locked)])), + ); + + assert_eq!(prebuild, after_deploy); + } + #[test] fn contained_relative_path_rejects_traversal_and_absolute() { assert!(is_contained_relative_path("u/admin/pkg")); @@ -6161,7 +6435,8 @@ mount {{ | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Nativets - | ScriptLang::Go => "//", + | ScriptLang::Go + | ScriptLang::Php => "//", _ => "", }; let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); @@ -6214,7 +6489,7 @@ mount {{ .await; if let Connection::Sql(db) = conn { - volume_setup = crate::volume_oss::setup_volumes_sql_worker( + volume_setup = Box::pin(crate::volume_oss::setup_volumes_sql_worker( &volume_mounts, db, &job.workspace_id, @@ -6227,10 +6502,10 @@ mount {{ language, &mut envs, &mut shared_mount, - ) + )) .await?; } else if let Connection::Http(http) = conn { - volume_setup = crate::volume_oss::setup_volumes_http_worker( + volume_setup = Box::pin(crate::volume_oss::setup_volumes_http_worker( &volume_mounts, http, &job.workspace_id, @@ -6243,7 +6518,7 @@ mount {{ language, &mut envs, &mut shared_mount, - ) + )) .await?; } } @@ -6380,6 +6655,7 @@ mount {{ envs, occupancy_metrics, maybe_lock, + modules.as_ref(), )) .await } @@ -6509,6 +6785,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules.as_ref(), )) .await } @@ -6567,6 +6844,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules.as_ref(), )) .await } @@ -6631,6 +6909,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules: modules.as_ref(), })) .await } @@ -6735,7 +7014,7 @@ mount {{ if let Some(ref vol_client) = volume_setup.client { if let Connection::Sql(db) = conn { - crate::volume_oss::sync_volumes_sql_worker( + Box::pin(crate::volume_oss::sync_volumes_sql_worker( &volume_setup.states, &volume_setup.writable, vol_client, @@ -6745,13 +7024,13 @@ mount {{ worker_name, conn, result.is_ok(), - ) + )) .await; } } if let Connection::Http(http) = conn { - crate::volume_oss::sync_volumes_http_worker( + Box::pin(crate::volume_oss::sync_volumes_http_worker( &volume_setup.states, &volume_setup.writable, http, @@ -6760,7 +7039,7 @@ mount {{ worker_name, conn, result.is_ok(), - ) + )) .await; } @@ -6795,7 +7074,7 @@ mount {{ result } -pub fn parse_sig_of_lang( +pub async fn parse_sig_of_lang( code: &str, language: Option<&ScriptLang>, main_override: Option, @@ -6830,10 +7109,9 @@ pub fn parse_sig_of_lang( ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?), ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?), #[cfg(feature = "php")] - ScriptLang::Php => Some(windmill_parser_php::parse_php_signature( - code, - main_override, - )?), + ScriptLang::Php => { + Some(crate::php_executor::parse_php_signature(code, main_override).await?) + } #[cfg(not(feature = "php"))] ScriptLang::Php => None, #[cfg(feature = "rust")] diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index bd39afa8c7..dbdba3e6ed 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -69,10 +69,10 @@ use windmill_common::{ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, - insert_concurrency_key_capped, interpolate_args, - report_error_to_workspace_handler_or_critical_side_channel, try_schedule_next_job, CanceledBy, - FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, - WrappedError, + insert_concurrency_key_capped, interpolate_args, render_tag_path, + report_error_to_workspace_handler_or_critical_side_channel, tag_reads_args, + tag_reads_flow_expr, try_schedule_next_job, CanceledBy, FlowRunners, MiniCompletedJob, + MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, RE_FLOW_EXPR_TAG, }; use windmill_audit::audit_oss::audit_log; @@ -2005,7 +2005,8 @@ pub async fn update_flow_status_after_job_completion_internal( if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); - let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; + let cached_res_path = + cached_result_path(db, client, &flow_job, Some(&flow)).await?; save_in_cache( db, @@ -2034,8 +2035,8 @@ pub async fn update_flow_status_after_job_completion_internal( chat_ai_info.conversation_id, ) .await?; - let (duration, wac_job_ids) = if success { - let (_, duration, wac_job_ids) = add_completed_job( + let duration = if success { + let (_, duration) = add_completed_job( db, &cflow_job, true, @@ -2049,9 +2050,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - (duration, wac_job_ids) + duration } else { - let (_, duration, wac_job_ids) = add_completed_job( + let (_, duration) = add_completed_job( db, &cflow_job, false, @@ -2069,30 +2070,11 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - (duration, wac_job_ids) + duration }; flow_job_duration = flow_job .started_at .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); - - // If this flow is a WAC child (not a flow step, has parent), - // notify the WAC parent of completion. - if !flow_job.is_flow_step() { - if let Some(parent_job) = flow_job.parent_job { - if let Some(job_ids) = wac_job_ids { - let _ = crate::result_processor::handle_wac_child_completion( - db, - &flow_job.id, - parent_job, - &flow_job.workspace_id, - nresult.clone(), - success, - job_ids, - ) - .await; - } - } - } } true } else { @@ -2254,6 +2236,7 @@ async fn add_tool_message_to_conversation( MessageType::Assistant, None, success, + None, ) .await?; tx.commit().await?; @@ -3132,6 +3115,66 @@ fn resolve_flow_step_tag( } } +/// Resolves each `$flow_expr[root.key.path]` of a step tag by reading `key.path` from `results` +/// (where `key` is a step id), `flow_input` or `flow_env`, rendered as `$args[key.path]` would be. +async fn interpolate_flow_expr_tag( + tag: &str, + db: &DB, + flow_job: &MiniPulledJob, + flow_input: &HashMap>, + flow_env: Option<&HashMap>>, +) -> error::Result { + if RE_FLOW_EXPR_TAG + .replace_all(tag, "") + .contains("$flow_expr[") + { + return Err(Error::ExecutionErr(format!( + "Could not resolve the step tag `{tag}`: each `$flow_expr[...]` must hold a dotted \ + path such as `results.a.b.c`" + ))); + } + let mut rendered: HashMap<&str, String> = HashMap::new(); + for cap in RE_FLOW_EXPR_TAG.captures_iter(tag) { + let path = cap.get(1).unwrap().as_str(); + if rendered.contains_key(path) { + continue; + } + let (root, key_path) = path.split_once('.').unwrap_or((path, "")); + let (key, rest) = key_path.split_once('.').unwrap_or((key_path, "")); + if key.is_empty() || !matches!(root, "results" | "flow_input" | "flow_env") { + return Err(Error::ExecutionErr(format!( + "Could not resolve the step tag `{tag}`: `{path}` must start with \ + `results.`, `flow_input.` or `flow_env.`" + ))); + } + let value = match root { + "flow_input" => render_tag_path(flow_input.get(key).map(|x| &**x), rest), + "flow_env" => render_tag_path(flow_env.and_then(|e| e.get(key)).map(|x| &**x), rest), + _ => match windmill_queue::get_result_by_id( + db.clone(), + flow_job.workspace_id.clone(), + flow_job.id, + key.to_string(), + None, + ) + .await + { + Ok(result) => render_tag_path(Some(&*result), rest), + Err(Error::NotFound(_)) => String::new(), + Err(e) => { + return Err(Error::ExecutionErr(format!( + "Could not resolve the step tag `{tag}`: {e}" + ))) + } + }, + }; + rendered.insert(path, value); + } + Ok(RE_FLOW_EXPR_TAG + .replace_all(tag, |cap: ®ex::Captures| rendered[&cap[1]].clone()) + .into_owned()) +} + #[cfg(test)] mod tag_resolution_tests { use super::resolve_flow_step_tag; @@ -3891,7 +3934,18 @@ async fn push_next_flow_job( drop(resume_messages); - let is_skipped = if let Some(skip_if) = &module.skip_if { + // `skip_if` is a one-time entry gate, so only first-entry statuses evaluate it. + // Once the module is looping, the last completed job is an inner iteration, not + // `previous_id`'s, and re-evaluating would alias `results.` to it. + // A restart-at-iteration also enters as `InProgress`: it resumes without re-gating. + let is_skipped = if let Some(skip_if) = module.skip_if.as_ref().filter(|_| { + matches!( + status_module, + FlowStatusModule::WaitingForPriorSteps { .. } + | FlowStatusModule::WaitingForEvents { .. } + | FlowStatusModule::WaitingForExecutor { .. } + ) + }) { let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status); let skip_if_res = compute_bool_from_expr( &skip_if.expr, @@ -4199,6 +4253,10 @@ async fn push_next_flow_job( None }; + // The `flow_input` the step's input transforms read, which a `$flow_expr[flow_input...]` + // tag must read too: the body of a simple for-loop also sees `iter` there. + let mut step_flow_input = arc_flow_job_args.clone(); + let marc; let me; let args = match &next_status { @@ -4223,8 +4281,10 @@ async fn push_next_flow_job( if let Some(input_transforms) = simple_input_transforms { //previous id is none because we do not want to use previous id if we are in a for loop let ctx = get_transform_context(&flow_job, "", &status); + let args = Marc::new(args); + step_flow_input = args.clone(); let ti = transform_input( - Marc::new(args), + args, flow_env, arc_last_job_result.clone(), input_transforms, @@ -4403,6 +4463,27 @@ async fn push_next_flow_job( payload_tag.tag.as_deref(), ); + // A step whose inputs failed to evaluate, or whose `$flow_expr[...]` tag failed to resolve, + // is pushed only to report the error, and a computed tag can then name a queue nobody + // serves (`push_args` is empty, so `$args[...]` reads nothing), leaving the step stuck + // instead. Send it to the flow's tag, which a worker is provably serving right now. + // + // A step handed over by id, or one whose tag `push` replaces, never reaches a worker + // through its tag, so rewriting theirs would be noise. + let step_is_pulled_by_tag = !continue_on_same_worker + && !continue_with_runners + && !payload_tag.payload.is_dedicated_worker(); + let reroute_to_flow_tag = err.is_some() + && step_is_pulled_by_tag + && tag + .as_deref() + .is_some_and(|t| tag_reads_args(t) || tag_reads_flow_expr(t)); + let tag = if reroute_to_flow_tag { + Some(flow_job.tag.clone()) + } else { + tag + }; + let (email, permissioned_as) = if let Some(on_behalf_of) = payload_tag.on_behalf_of.as_ref() { (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) @@ -4421,8 +4502,9 @@ async fn push_next_flow_job( .as_deref() .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) { - let is_super_admin = - windmill_common::auth::is_super_admin_email(db, email).await?; + // A step with its own on-behalf-of carries a cached dispatch address, up to one + // notify poll stale; accepted, see `get_email_from_permissioned_as`. + let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; check_tag_available_for_workspace_internal( db, &flow_job.workspace_id, @@ -4437,6 +4519,30 @@ async fn push_next_flow_job( .await?; } + // Resolved only after the check: CUSTOM_TAGS allows the template, so its value may name + // any queue, as the value of an `$args[...]` tag does. + let mut tag_err = None; + let tag = match tag { + Some(t) if err.is_none() && tag_reads_flow_expr(&t) => { + match interpolate_flow_expr_tag(&t, db, &flow_job, &step_flow_input, flow_env) + .warn_after_seconds(3) + .await + { + Ok(resolved) => Some(resolved), + Err(e) => { + tag_err = Some(e); + Some(if step_is_pulled_by_tag { + flow_job.tag.clone() + } else { + t + }) + } + } + } + t => t, + }; + let err = err.or(tag_err.as_ref()); + let evaluated_timeout = if let Some(timeout_transform) = &module.timeout { let ctx = get_transform_context(&flow_job, &previous_id, &status); @@ -6155,9 +6261,7 @@ pub async fn script_to_payload( .await? .prefetch_cached(&db) .await?; - let on_behalf_of = script_info - .on_behalf_of(&flow_job.workspace_id, db) - .await?; + let on_behalf_of = script_info.on_behalf_of(&flow_job.workspace_id, db).await?; let ScriptHashInfo { tag, cache_ttl, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 5cfb9f08ce..03150beb00 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -19,14 +19,17 @@ use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::jobs::JobKind; use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; -use windmill_common::scripts::ScriptHash; +use windmill_common::scripts::{ + deploy_relocked_version, fetch_script_for_update, hash_script, ScriptHash, ScriptModule, +}; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; -use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; +use windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -38,8 +41,10 @@ use windmill_common::{ scripts::ScriptLang, DB, }; +use windmill_dep_map::lock_hash::record_lock_hashes; pub use windmill_dep_map::{ extract_referenced_paths, extract_relative_imports, process_relative_imports, + refresh_dependency_map, }; use windmill_git_sync::{ handle_deployment_metadata, tally_deployed_object_changes, DeployedObject, @@ -86,10 +91,13 @@ use crate::{ /// has the toolchain and, since the cache key is per OS/arch, the platform the runtime /// workers use. Deploys that supply their own lock never reach a dependency job at all and /// queue theirs from `create_script_internal` instead. -async fn maybe_queue_binary_prebuild(db: &DB, job: &MiniPulledJob, lock: &str) -> Result<()> { - let (Some(hash), Some(path), Some(lang)) = - (job.runnable_id, job.runnable_path.clone(), job.script_lang) - else { +async fn maybe_queue_binary_prebuild( + db: &DB, + job: &MiniPulledJob, + hash: ScriptHash, + lock: &str, +) -> Result<()> { + let (Some(path), Some(lang)) = (job.runnable_path.clone(), job.script_lang) else { return Ok(()); }; let Some(prebuild) = @@ -166,7 +174,7 @@ async fn handle_build_binary_job( if !crate::global_cache::object_store_available().await { return Ok(to_raw_value_owned(json!({ "status": "skipped", - "reason": "this worker cannot reach the instance object store, so the binary \ + "reason": "this worker cannot reach an object store, so the binary \ would not be shared with other workers", }))); } @@ -199,6 +207,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -214,6 +223,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -235,6 +245,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -280,6 +291,13 @@ pub async fn handle_dependency_job( job.runnable_path() ); let script_path = job.runnable_path(); + let w_id = &job.workspace_id; + + let triggered_by_relative_import = job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default(); // A build pass reads the same script data but writes none of the deploy state below, // including the `lock_error_logs` stamp on a fetch failure: the version it builds is @@ -293,14 +311,48 @@ pub async fn handle_dependency_job( *deployment_tallied = true; } + // A relative-import relock deploys nothing until `commit_relock` says so, while the + // caller's fallback tally assumes a failed dependency job left a deployed version behind. + // Claim the tally here; the failure path hands it back once it has minted the version + // that carries the error. + if triggered_by_relative_import { + *deployment_tallied = true; + } + + // A relative-import relock locks the path's live version as of now, not the hash captured + // when the job was pushed: a deploy can land during the debounce delay, after which that + // hash names an archived version. What it generates is committed against the live version + // re-read under a row lock, so a deploy landing mid-generation is caught there too. + let target_hash = if triggered_by_relative_import { + Some(ScriptHash(live_head_hash(db, w_id, script_path).await?)) + } else { + job.runnable_id + }; + // `JobKind::Dependencies` job store either: // - A saved script `hash` in the `script_hash` column. // - Preview raw lock and code in the `queue` or `job` table. - let script_data = &match job.runnable_id { + let script_data = &match target_hash { + // Read straight from the database: the cache pins a version's data under its hash for + // as long as this worker lives, and the live version may still be waiting on its own + // dependency job's lock, which lands in place. A run resolving to it on this worker + // would then get no lock from the cache at all. + Some(hash) if triggered_by_relative_import => { + let raw = cache::script::fetch_script_from_db(db, hash, std::panic::Location::caller()) + .await?; + Cow::Owned(std::sync::Arc::new(cache::ScriptData { + lock: raw.lock, + code: raw.content, + modules: raw.modules, + })) + } Some(hash) => match cache::script::fetch(&Connection::from(db.clone()), hash).await { Ok(d) => Cow::Owned(d.0), Err(e) => { - if !is_build_job { + // The live version of a relative-import relock is what runs resolve to, and + // `lock_error_logs` on it takes it out of resolution; the job carries the + // error instead, since it deployed nothing. + if !is_build_job && !triggered_by_relative_import { let logs2 = sqlx::query_scalar!( "SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2", &job.id, @@ -345,12 +397,6 @@ pub async fn handle_dependency_job( .await; } - let triggered_by_relative_import = job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default(); - // Extract temp_script_refs from job args (path -> hash mapping for temp storage) let temp_script_refs: Option> = job .args @@ -388,20 +434,17 @@ pub async fn handle_dependency_job( ) .await; + let (deployment_message, parent_path) = + get_deployment_msg_and_parent_path_from_args(job.args.clone()); + match content { Ok(content) => { - if job.runnable_id.is_none() { + let Some(current_hash) = target_hash else { // it a one-off raw script dependency job, no need to update the db return Ok(to_raw_value_owned( json!({ "status": "Successful lock file generation", "lock": content }), )); - } - - let current_hash = job.runnable_id.unwrap_or(ScriptHash(0)); - let w_id = &job.workspace_id; - - let (deployment_message, parent_path) = - get_deployment_msg_and_parent_path_from_args(job.args.clone()); + }; // Generate lockfiles for module files (if any). // @@ -438,7 +481,9 @@ pub async fn handle_dependency_job( occupancy_metrics, &raw_workspace_dependencies_o, module.lock.as_deref(), - triggered_by_relative_import, + // A module that was never locked has nothing a skip could hand + // back; the path's lock is the parent script's, not its own. + triggered_by_relative_import && module.lock.is_some(), script_path, None, "script", @@ -461,34 +506,83 @@ pub async fn handle_dependency_job( None }; - // We do not create new row for this update - // That means we can keep current hash and just update lock - // Also store lockfile hash for dependency change detection - let lockfile_hash = windmill_common::scripts::hash_script(&content); - let updated_modules_json = updated_modules - .as_ref() - .and_then(|m| serde_json::to_value(m).ok()); - sqlx::query!( - "WITH update_lock AS ( - UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + let deployed_hash = if triggered_by_relative_import { + match commit_relock( + db, + w_id, + script_path, + current_hash, + Some(&content), + updated_modules.as_ref(), + None, + deployment_message.clone(), ) - INSERT INTO lock_hash (workspace_id, path, lockfile_hash) - VALUES ($3, $4, $5) - ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", - &content, - ¤t_hash.0, - w_id, - script_path, - &lockfile_hash, - updated_modules_json - ) - .execute(db) - .await?; + .await? + { + RelockOutcome::Deployed(hash) => hash, + RelockOutcome::Unchanged => { + let log_msg = "\nLock unchanged: no new version deployed"; + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + // The imports may have moved even though the result did not, and the + // map is what this importer's next skip check reads. + refresh_dependency_map( + db, + w_id, + script_path, + &parent_path, + &script_data.code, + &job.script_lang, + ) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock unchanged, no new version deployed", "lock": content }), + )); + } + RelockOutcome::Superseded(head) => { + let log_msg = format!( + "\nVersion {head} was deployed while this lock was generated; discarding it and queueing a relock of that version" + ); + tracing::info!(workspace_id = %w_id, job_id = %job.id, "{log_msg}"); + append_logs(&job.id, w_id, log_msg, &db.into()).await; + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await?; + return Ok(to_raw_value_owned( + json!({ "status": "Lock generation superseded by a newer version", "lock": content }), + )); + } + } + } else { + // We do not create new row for this update + // That means we can keep current hash and just update lock + // Also store lockfile hash for dependency change detection + let lockfile_hash = windmill_common::scripts::hash_script(&content); + let updated_modules_json = updated_modules + .as_ref() + .and_then(|m| serde_json::to_value(m).ok()); + sqlx::query!( + "WITH update_lock AS ( + UPDATE script SET lock = $1, modules = COALESCE($6, modules) WHERE hash = $2 AND workspace_id = $3 + ) + INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + VALUES ($3, $4, $5) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = $5", + &content, + ¤t_hash.0, + w_id, + script_path, + &lockfile_hash, + updated_modules_json + ) + .execute(db) + .await?; - // `lock` has been updated; invalidate the cache. - // Since only worker that ran this Dependency Job has the cache - // we do not need to think about invalidating cache for other workers. - cache::script::invalidate(current_hash); + // `lock` has been updated; invalidate the cache. + // Since only worker that ran this Dependency Job has the cache + // we do not need to think about invalidating cache for other workers. + cache::script::invalidate(current_hash); + current_hash + }; // The version only became runnable now, so this process still resolves the path to // the one before it. Only the runnable-hash cache: the import-side caches ignore the // lock, so evicting this process' half of that pair here would key a bundle by a @@ -501,7 +595,7 @@ pub async fn handle_dependency_job( &db, &w_id, DeployedObject::Script { - hash: current_hash, + hash: deployed_hash, path: script_path.to_string(), parent_path: parent_path.clone(), }, @@ -562,7 +656,7 @@ pub async fn handle_dependency_job( }); } - if let Err(e) = maybe_queue_binary_prebuild(db, job, &content).await { + if let Err(e) = maybe_queue_binary_prebuild(db, job, deployed_hash, &content).await { tracing::error!(%e, "error queueing the auto-build binary job for {script_path}"); } @@ -580,14 +674,49 @@ pub async fn handle_dependency_job( .await? .flatten() .unwrap_or_else(|| "no logs".to_string()); - sqlx::query!( - "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", - &format!("{logs2}\n{error}"), - &job.runnable_id.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; + let error_logs = format!("{logs2}\n{error}"); + if let (true, Some(hash)) = (triggered_by_relative_import, target_hash) { + // The same shape a failed deploy leaves: a version without a lock that carries + // the error, so it shows on the script while runs keep resolving to the last + // version that has one. Only that version is the caller's fallback to tally; + // one that landed meanwhile owns its own lock, and a commit that failed left + // nothing. + match commit_relock( + db, + w_id, + script_path, + hash, + None, + None, + Some(&error_logs), + deployment_message.clone(), + ) + .await + { + Ok(RelockOutcome::Deployed(_)) => *deployment_tallied = false, + Ok(RelockOutcome::Superseded(_)) => { + if let Err(e) = + requeue_relock(db, job, script_path, deployment_message, parent_path) + .await + { + tracing::error!(%e, "error queueing a relock of {script_path}") + } + } + Ok(RelockOutcome::Unchanged) => {} + Err(e) => { + tracing::error!(%e, "error recording the failed relock of {script_path}") + } + } + } else { + sqlx::query!( + "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", + &error_logs, + &job.runnable_id.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + } Err(Error::ExecutionErr(format!( "Error locking file: {error}\n\nlogs:\n{}", remove_ansi_codes(&logs2) @@ -595,6 +724,136 @@ pub async fn handle_dependency_job( } } } + +/// The version of `script_path` that runs resolve to, which is what a relative-import relock +/// locks. `NotFound` when the path holds none, which a job pushed for a path since archived or +/// deleted reports as its own failure. +async fn live_head_hash(db: &DB, w_id: &str, script_path: &str) -> error::Result { + sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + )) + }) +} + +enum RelockOutcome { + /// A new version carrying the result is the live one. + Deployed(ScriptHash), + /// The live version already holds this lock and these module locks; nothing was written. + Unchanged, + /// The live version is no longer the one the lock was generated for; nothing was written. + Superseded(ScriptHash), +} + +/// Commits what a relative-import relock produced against the path's live version, read under +/// a row lock so relocks of one path serialize on it. +/// +/// A result equal to the live version's lock and module locks writes nothing: the importer's +/// dependencies did not move, and a new version would deploy byte-identical content and then +/// walk its own importers for nothing. A `lock` of `None` is a failed generation and always +/// deploys, as the version that carries the error. +async fn commit_relock( + db: &DB, + w_id: &str, + script_path: &str, + generated_for: ScriptHash, + lock: Option<&str>, + modules: Option<&HashMap>, + lock_error_logs: Option<&str>, + deployment_message: Option, +) -> error::Result { + let mut tx = db.begin().await?; + let mut head = None; + for _ in 0..4 { + head = fetch_script_for_update(script_path, w_id, &mut *tx).await?; + if head.is_some() { + break; + } + // Having waited on the live version's row lock, the statement re-checked that row + // once the holder committed, found it archived, and returned nothing: the successor + // the holder inserted is not in the statement's snapshot. A fresh statement sees it, + // unless yet another writer got there first, so this goes around a few times before + // concluding the path holds no live version. + } + let Some(head) = head else { + return Err(Error::NotFound(format!( + "Non-archived script with path '{script_path}' not found" + ))); + }; + if head.hash != generated_for { + // A deploy landed while the lock was generated. It carried its own lock or queued its + // own dependency job, and this lock describes content that is no longer live. + return Ok(RelockOutcome::Superseded(head.hash)); + } + let lock_hash_entry = lock.map(|lock| (script_path.to_string(), hash_script(lock))); + if let Some(lock) = lock { + let modules_unchanged = modules.map_or(true, |m| head.modules.as_ref() == Some(m)); + if head.lock.as_deref() == Some(lock) && modules_unchanged { + // The hash row is still written, and under the same row lock: a version deployed + // before lock hashes were recorded has none, so its importers cannot skip until + // it does, and a deploy that takes the lock next must not have the hash it records + // overwritten by this one. + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + return Ok(RelockOutcome::Unchanged); + } + } + let new_hash = deploy_relocked_version( + &mut tx, + head, + deployment_message, + lock, + modules, + lock_error_logs, + ) + .await?; + record_lock_hashes(&mut tx, w_id, lock_hash_entry.as_slice()).await?; + tx.commit().await?; + Ok(RelockOutcome::Deployed(ScriptHash(new_hash))) +} + +/// Queues another relative-import relock of `script_path`, through the same push the fan-out +/// uses. The version live now was deployed while a lock was generated for its predecessor; +/// when that deploy was a sibling relock it queued nothing for this path, and the result just +/// discarded may have been the one generated against the current imports. +async fn requeue_relock( + db: &DB, + job: &MiniPulledJob, + script_path: &str, + deployment_message: Option, + parent_path: Option, +) -> error::Result<()> { + let already_visited = job + .args + .as_ref() + .and_then(|x| x.get("already_visited")) + .and_then(|v| serde_json::from_str::>(v.get()).ok()) + .unwrap_or_default(); + trigger_dependents_to_recompute_dependencies( + &job.workspace_id, + vec![DependencyDependent { + importer_path: script_path.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + }], + deployment_message, + parent_path, + &job.permissioned_as_email, + &job.created_by, + &job.permissioned_as, + db, + already_visited, + ) + .await +} + fn remove_ansi_codes(s: &str) -> String { lazy_static::lazy_static! { static ref ANSI_REGEX: regex::Regex = regex::Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").unwrap(); @@ -1536,6 +1795,17 @@ async fn lock_modules( agent, tool_inputs, } => { + if let Some(agent_path) = agent.as_deref().filter(|_| !skip_flow_update) { + sqlx::query!( + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + job_path, + agent_path, + job.workspace_id, + ) + .execute(db) + .await?; + } + // Extract FlowModules from tools and track their original indices // MCP tools don't need locking, so we filter them out let mut flow_modules = Vec::new(); @@ -2889,8 +3159,12 @@ async fn try_skip_relock( } // Fetch existing lock based on runnable type - let lock = match runnable_type { - "script" => sqlx::query_scalar!( + let lock = match (runnable_type, existing_lock) { + // A script's module asks with the script's own type and hands over the lock it last + // deployed with. The path's lock below is the parent script's, and a module given + // that loses whatever it resolves on its own. + ("script", Some(module_lock)) => Some(module_lock.to_string()), + ("script", None) => sqlx::query_scalar!( "SELECT lock FROM script WHERE path = $1 AND workspace_id = $2 AND lock IS NOT NULL AND deleted = false ORDER BY created_at DESC LIMIT 1", base_path, @@ -2900,7 +3174,7 @@ async fn try_skip_relock( .await? .flatten(), - "flow" | "app" => existing_lock.map(|s| s.to_string()), + ("flow" | "app", existing_lock) => existing_lock.map(|s| s.to_string()), _ => None, }; @@ -3155,7 +3429,11 @@ async fn capture_dependency_job( ) .await? { - if !wd_exist { + // Nothing here writes the module files, so a bundle built now resolves a + // multi-file script's relative imports remotely rather than from its + // modules; caching that under a key naming them would serve the wrong + // code to every run. Leave it to the first run instead. + if !wd_exist && modules.map_or(true, |m| m.is_empty()) { crate::bun_executor::prebundle_bun_script( job_raw_code, &lock, @@ -3169,6 +3447,7 @@ async fn capture_dependency_job( &token, &mut Some(occupancy_metrics), temp_script_refs, + modules, ) .await?; } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 96c29cb209..7f0649a2ab 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.798.1"; +export const VERSION = "v1.816.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/benchmarks/pulumi/package-lock.json b/benchmarks/pulumi/package-lock.json deleted file mode 100644 index ddd1f09e83..0000000000 --- a/benchmarks/pulumi/package-lock.json +++ /dev/null @@ -1,2465 +0,0 @@ -{ - "name": "aws-bench", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "aws-bench", - "dependencies": { - "@pulumi/aws": "^5.0.0", - "@pulumi/awsx": "^1.0.4", - "@pulumi/pulumi": "^3.0.0", - "@pulumi/tailscale": "^0.12.2", - "@pulumi/tls": "^4.10.0" - }, - "devDependencies": { - "@types/node": "^16", - "pulumi": "^0.0.1" - } - }, - "node_modules/@grpc/grpc-js": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", - "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/proto-loader": "^0.8.0", - "@js-sdsl/ordered-map": "^4.4.2" - }, - "engines": { - "node": ">=12.10.0" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", - "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.5.3", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@js-sdsl/ordered-map": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", - "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/js-sdsl" - } - }, - "node_modules/@logdna/tail-file": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@logdna/tail-file/-/tail-file-2.2.0.tgz", - "integrity": "sha512-XGSsWDweP80Fks16lwkAUIr54ICyBs6PsI4mpfTLQaWgEJRtY9xEV+PeyDpJ+sJEGZxqINlpmAwe/6tS1pP8Ng==", - "engines": { - "node": ">=10.3.0" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", - "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-metrics": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-metrics/-/api-metrics-0.32.0.tgz", - "integrity": "sha512-g1WLhpG8B6iuDyZJFRGsR+JKyZ94m5LEmY2f+duEJ9Xb4XRlLHrZvh6G34OH6GJ8iDHxfHb/sWjJ1ZpkI9yGMQ==", - "deprecated": "Please use @opentelemetry/api >= 1.3.0", - "dependencies": { - "@opentelemetry/api": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.15.2.tgz", - "integrity": "sha512-VAMHG67srGFQDG/N2ns5AyUT9vUcoKpZ/NpJ5fDQIPfJd7t3ju+aHwvDsMcrYBWuCh03U3Ky6o16+872CZchBg==", - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.15.2.tgz", - "integrity": "sha512-+gBv15ta96WqkHZaPpcDHiaz0utiiHZVfm2YOYSqFGrUaJpPkMoSuLBB58YFQGi6Rsb9EHos84X6X5+9JspmLw==", - "dependencies": { - "@opentelemetry/semantic-conventions": "1.15.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/exporter-zipkin": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.15.2.tgz", - "integrity": "sha512-j9dPe8tyx4KqIqJAfZ/LCYfkF9+ggsT0V1+bVg9ZKTBNcLf5dTsTMdcxUxc/9s599kgcn6UERnti/tozbzwa6Q==", - "dependencies": { - "@opentelemetry/core": "1.15.2", - "@opentelemetry/resources": "1.15.2", - "@opentelemetry/sdk-trace-base": "1.15.2", - "@opentelemetry/semantic-conventions": "1.15.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.32.0.tgz", - "integrity": "sha512-y6ADjHpkUz/v1nkyyYjsQa/zorhX+0qVGpFvXMcbjU4sHnBnC02c6wcc93sIgZfiQClIWo45TGku1KQxJ5UUbQ==", - "dependencies": { - "@opentelemetry/api-metrics": "0.32.0", - "require-in-the-middle": "^5.0.3", - "semver": "^7.3.2", - "shimmer": "^1.2.1" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/instrumentation-grpc": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-grpc/-/instrumentation-grpc-0.32.0.tgz", - "integrity": "sha512-Az6wdkPx/Mi26lT9LKFV6GhCA9prwQFPz5eCNSExTnSP49YhQ7XCjzPd2POPeLKt84ICitrBMdE1mj0zbPdLAQ==", - "dependencies": { - "@opentelemetry/api-metrics": "0.32.0", - "@opentelemetry/instrumentation": "0.32.0", - "@opentelemetry/semantic-conventions": "1.6.0" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.0.0" - } - }, - "node_modules/@opentelemetry/instrumentation-grpc/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.6.0.tgz", - "integrity": "sha512-aPfcBeLErM/PPiAuAbNFLN5sNbZLc3KZlar27uohllN8Zs6jJbHyJU1y7cMA6W/zuq+thkaG8mujiS+3iD/FWQ==", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/instrumentation/node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@opentelemetry/propagator-b3": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.15.2.tgz", - "integrity": "sha512-ZSrL3DpMEDsjD8dPt9Ze3ue53nEXJt512KyxXlLgLWnSNbe1mrWaXWkh7OLDoVJh9LqFw+tlvAhDVt/x3DaFGg==", - "dependencies": { - "@opentelemetry/core": "1.15.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/propagator-jaeger": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.15.2.tgz", - "integrity": "sha512-6m1yu7PVDIRz6BwA36lacfBZJCfAEHKgu+kSyukNwVdVjsTNeyD9xNPQnkl0WN7Rvhk8/yWJ83tLPEyGhk1wCQ==", - "dependencies": { - "@opentelemetry/core": "1.15.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.15.2.tgz", - "integrity": "sha512-xmMRLenT9CXmm5HMbzpZ1hWhaUowQf8UB4jMjFlAxx1QzQcsD3KFNAVX/CAWzFPtllTyTplrA4JrQ7sCH3qmYw==", - "dependencies": { - "@opentelemetry/core": "1.15.2", - "@opentelemetry/semantic-conventions": "1.15.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.15.2.tgz", - "integrity": "sha512-BEaxGZbWtvnSPchV98qqqqa96AOcb41pjgvhfzDij10tkBhIu9m0Jd6tZ1tJB5ZHfHbTffqYVYE0AOGobec/EQ==", - "dependencies": { - "@opentelemetry/core": "1.15.2", - "@opentelemetry/resources": "1.15.2", - "@opentelemetry/semantic-conventions": "1.15.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-node": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.15.2.tgz", - "integrity": "sha512-5deakfKLCbPpKJRCE2GPI8LBE2LezyvR17y3t37ZI3sbaeogtyxmBaFV+slmG9fN8OaIT+EUsm1QAT1+z59gbQ==", - "dependencies": { - "@opentelemetry/context-async-hooks": "1.15.2", - "@opentelemetry/core": "1.15.2", - "@opentelemetry/propagator-b3": "1.15.2", - "@opentelemetry/propagator-jaeger": "1.15.2", - "@opentelemetry/sdk-trace-base": "1.15.2", - "semver": "^7.5.1" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-node/node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.15.2.tgz", - "integrity": "sha512-CjbOKwk2s+3xPIMcd5UNYQzsf+v94RczbdNix9/kQh38WiQkM90sUOi3if8eyHFgiBjBjhwXrA7W3ydiSQP9mw==", - "engines": { - "node": ">=14" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", - "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", - "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", - "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", - "license": "BSD-3-Clause" - }, - "node_modules/@pulumi/aws": { - "version": "5.43.0", - "resolved": "https://registry.npmjs.org/@pulumi/aws/-/aws-5.43.0.tgz", - "integrity": "sha512-ZWI+QvEaFy27wUW8j8EVadgvl6u3926AAgpghflSPxF/9mG5XoDy0rqkRZOgO13ZJinC1qkQDzoMz0ACgmwTiw==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@pulumi/pulumi": "^3.0.0", - "aws-sdk": "^2.0.0", - "builtin-modules": "3.0.0", - "mime": "^2.0.0", - "read-package-tree": "^5.2.1", - "resolve": "^1.7.1" - } - }, - "node_modules/@pulumi/awsx": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@pulumi/awsx/-/awsx-1.0.6.tgz", - "integrity": "sha512-zTBsRO6EeSg6V2sEkw1b8OtvExaYMg1rUlaJVfui54W86M3KvFANtGI99a+aScUxJMxGqkS9cb6LdzZjTlKQGA==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "@pulumi/aws": "^5.35.0", - "@pulumi/docker": "^3.6.1", - "@pulumi/pulumi": "^3.0.0", - "@types/aws-lambda": "^8.10.23", - "mime": "^2.0.0" - } - }, - "node_modules/@pulumi/docker": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/@pulumi/docker/-/docker-3.6.1.tgz", - "integrity": "sha512-BZME50QkT556v+LvmTXPT8ssB2xxNkp9+msB5xYFEnUnWcdGAx5yUysQw70RJCb+U0GbkJSbxtlgMJgOQf/now==", - "hasInstallScript": true, - "dependencies": { - "@pulumi/pulumi": "^3.0.0", - "semver": "^5.4.0" - } - }, - "node_modules/@pulumi/pulumi": { - "version": "3.77.1", - "resolved": "https://registry.npmjs.org/@pulumi/pulumi/-/pulumi-3.77.1.tgz", - "integrity": "sha512-smeip4uKWkaKcNhMiAmR5uW4nXRvmHSjq93C7zx+mMneNxeaD9HNIBnYW0H0FFgp7j1AdRYuiIa2ie9Ay51bpw==", - "dependencies": { - "@grpc/grpc-js": "^1.8.16", - "@logdna/tail-file": "^2.0.6", - "@opentelemetry/api": "^1.2.0", - "@opentelemetry/exporter-zipkin": "^1.6.0", - "@opentelemetry/instrumentation": "^0.32.0", - "@opentelemetry/instrumentation-grpc": "^0.32.0", - "@opentelemetry/resources": "^1.6.0", - "@opentelemetry/sdk-trace-base": "^1.6.0", - "@opentelemetry/sdk-trace-node": "^1.6.0", - "@opentelemetry/semantic-conventions": "^1.6.0", - "@pulumi/query": "^0.3.0", - "execa": "^5.1.0", - "google-protobuf": "^3.5.0", - "ini": "^2.0.0", - "js-yaml": "^3.14.0", - "minimist": "^1.2.6", - "normalize-package-data": "^3.0.0", - "pkg-dir": "^7.0.0", - "read-package-tree": "^5.3.1", - "require-from-string": "^2.0.1", - "semver": "^7.5.2", - "source-map-support": "^0.5.6", - "ts-node": "^7.0.1", - "typescript": "~3.8.3", - "upath": "^1.1.0" - }, - "engines": { - "node": ">=8.13.0 || >=10.10.0" - } - }, - "node_modules/@pulumi/pulumi/node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@pulumi/query": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@pulumi/query/-/query-0.3.0.tgz", - "integrity": "sha512-xfo+yLRM2zVjVEA4p23IjQWzyWl1ZhWOGobsBqRpIarzLvwNH/RAGaoehdxlhx4X92302DrpdIFgTICMN4P38w==" - }, - "node_modules/@pulumi/tailscale": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@pulumi/tailscale/-/tailscale-0.12.2.tgz", - "integrity": "sha512-eDzXu4I7awua494d6is9nwV/8G56dTI4oL53G/WTqWtl1ezVHb+v6bKxMrcPvhhYGugbIhK56ECWPiqg9k5gJg==", - "hasInstallScript": true, - "dependencies": { - "@pulumi/pulumi": "^3.0.0" - } - }, - "node_modules/@pulumi/tls": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@pulumi/tls/-/tls-4.10.0.tgz", - "integrity": "sha512-aK1LXJzDbeVYBIlxycUUboZJSjMr2wIkrpngNRrOFPMIyO4QYNzJHWIwRbNkS463hhpbp1it3IC3htEqtP4lzg==", - "hasInstallScript": true, - "dependencies": { - "@pulumi/pulumi": "^3.0.0" - } - }, - "node_modules/@types/aws-lambda": { - "version": "8.10.119", - "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.119.tgz", - "integrity": "sha512-Vqm22aZrCvCd6I5g1SvpW151jfqwTzEZ7XJ3yZ6xaZG31nUEOEyzzVImjRcsN8Wi/QyPxId/x8GTtgIbsy8kEw==" - }, - "node_modules/@types/node": { - "version": "16.18.40", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.40.tgz", - "integrity": "sha512-+yno3ItTEwGxXiS/75Q/aHaa5srkpnJaH+kdkTVJ3DtJEwv92itpKbxU+FjPoh2m/5G9zmUQfrL4A4C13c+iGA==" - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/array-buffer-byte-length": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", - "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", - "dependencies": { - "call-bind": "^1.0.2", - "is-array-buffer": "^3.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/array.prototype.reduce": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/array.prototype.reduce/-/array.prototype.reduce-1.0.5.tgz", - "integrity": "sha512-kDdugMl7id9COE8R7MHF5jWk7Dqt/fs4Pv+JXoICnYwqpjjjbUurz6w5fT5IG6brLdJhv6/VoHB0H7oyIBXd+Q==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", - "es-array-method-boxes-properly": "^1.0.0", - "is-string": "^1.0.7" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/arraybuffer.prototype.slice": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.1.tgz", - "integrity": "sha512-09x0ZWFEjj4WD8PDbykUwo3t9arLn8NIzmmYEJFpYekOAQjpkGSyrQhNoRTcwwcFRu+ycWF78QZ63oWTqSjBcw==", - "dependencies": { - "array-buffer-byte-length": "^1.0.0", - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "get-intrinsic": "^1.2.1", - "is-array-buffer": "^3.0.2", - "is-shared-array-buffer": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/arrify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/asap": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" - }, - "node_modules/available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/aws-sdk": { - "version": "2.1693.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1693.0.tgz", - "integrity": "sha512-cJmb8xEnVLT+R6fBS5sn/EFJiX7tUnDaPtOPZ1vFbOJtd0fnZn/Ky2XGgsvvoeliWeH7mL3TWSX5zXXGSQV6gQ==", - "deprecated": "The AWS SDK for JavaScript (v2) has reached end-of-support, and no longer receives updates. Please migrate your code to use AWS SDK for JavaScript (v3). More info https://a.co/cUPnyil", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/brace-expansion": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", - "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" - }, - "node_modules/builtin-modules": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.0.0.tgz", - "integrity": "sha512-hMIeU4K2ilbXV6Uv93ZZ0Avg/M91RaKXucQ+4me2Do1txxBDyDZWCBa5bJSLqoNTRpXTLwEzIk1KmloenDDjhg==", - "engines": { - "node": ">=6" - } - }, - "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", - "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", - "engines": { - "node": "*" - } - }, - "node_modules/define-properties": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.0.tgz", - "integrity": "sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA==", - "dependencies": { - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/dezalgo": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", - "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", - "dependencies": { - "asap": "^2.0.0", - "wrappy": "1" - } - }, - "node_modules/diff": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.1.tgz", - "integrity": "sha512-Z3u54A8qGyqFOSr2pk0ijYs8mOE9Qz8kTvtKeBI+upoG9j04Sq+oI7W8zAJiQybDcESET8/uIdHzs0p3k4fZlw==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/es-abstract": { - "version": "1.22.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.1.tgz", - "integrity": "sha512-ioRRcXMO6OFyRpyzV3kE1IIBd4WG5/kltnzdxSCqoP8CMGs/Li+M1uF5o7lOkZVFjDs+NLesthnF66Pg/0q0Lw==", - "dependencies": { - "array-buffer-byte-length": "^1.0.0", - "arraybuffer.prototype.slice": "^1.0.1", - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "es-set-tostringtag": "^2.0.1", - "es-to-primitive": "^1.2.1", - "function.prototype.name": "^1.1.5", - "get-intrinsic": "^1.2.1", - "get-symbol-description": "^1.0.0", - "globalthis": "^1.0.3", - "gopd": "^1.0.1", - "has": "^1.0.3", - "has-property-descriptors": "^1.0.0", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "internal-slot": "^1.0.5", - "is-array-buffer": "^3.0.2", - "is-callable": "^1.2.7", - "is-negative-zero": "^2.0.2", - "is-regex": "^1.1.4", - "is-shared-array-buffer": "^1.0.2", - "is-string": "^1.0.7", - "is-typed-array": "^1.1.10", - "is-weakref": "^1.0.2", - "object-inspect": "^1.12.3", - "object-keys": "^1.1.1", - "object.assign": "^4.1.4", - "regexp.prototype.flags": "^1.5.0", - "safe-array-concat": "^1.0.0", - "safe-regex-test": "^1.0.0", - "string.prototype.trim": "^1.2.7", - "string.prototype.trimend": "^1.0.6", - "string.prototype.trimstart": "^1.0.6", - "typed-array-buffer": "^1.0.0", - "typed-array-byte-length": "^1.0.0", - "typed-array-byte-offset": "^1.0.0", - "typed-array-length": "^1.0.4", - "unbox-primitive": "^1.0.2", - "which-typed-array": "^1.1.10" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/es-array-method-boxes-properly": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz", - "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" - }, - "node_modules/es-set-tostringtag": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz", - "integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==", - "dependencies": { - "get-intrinsic": "^1.1.3", - "has": "^1.0.3", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dependencies": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/find-up": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", - "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", - "dependencies": { - "locate-path": "^7.1.0", - "path-exists": "^5.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dependencies": { - "is-callable": "^1.1.3" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" - }, - "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" - }, - "node_modules/function.prototype.name": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", - "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.3", - "es-abstract": "^1.19.0", - "functions-have-names": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/functions-have-names": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", - "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", - "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-symbol-description": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", - "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", - "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/globalthis": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", - "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", - "dependencies": { - "define-properties": "^1.1.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/google-protobuf": { - "version": "3.21.2", - "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.21.2.tgz", - "integrity": "sha512-3MSOYFO5U9mPGikIYCzK0SaThypfGgS6bHqrUGXG3DPHCrb+txNqeEcns1W0lkGfk0rCyNXm7xB9rMxnCiZOoA==" - }, - "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" - }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/has-bigints": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", - "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-property-descriptors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", - "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", - "dependencies": { - "get-intrinsic": "^1.1.1" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dependencies": { - "has-symbols": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hosted-git-info": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", - "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "node_modules/ini": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", - "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", - "engines": { - "node": ">=10" - } - }, - "node_modules/internal-slot": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", - "integrity": "sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==", - "dependencies": { - "get-intrinsic": "^1.2.0", - "has": "^1.0.3", - "side-channel": "^1.0.4" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-array-buffer": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", - "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", - "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.0", - "is-typed-array": "^1.1.10" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-bigint": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", - "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", - "dependencies": { - "has-bigints": "^1.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-boolean-object": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", - "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-core-module": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", - "integrity": "sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==", - "dependencies": { - "has": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-date-object": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", - "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-negative-zero": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", - "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-number-object": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", - "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-regex": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", - "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-shared-array-buffer": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", - "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", - "dependencies": { - "call-bind": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-string": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", - "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-symbol": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", - "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", - "dependencies": { - "has-symbols": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "dependencies": { - "which-typed-array": "^1.1.11" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-weakref": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", - "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", - "dependencies": { - "call-bind": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" - }, - "node_modules/locate-path": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", - "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", - "dependencies": { - "p-locate": "^6.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "license": "MIT" - }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" - }, - "node_modules/mime": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", - "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", - "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" - }, - "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/normalize-package-data": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", - "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", - "dependencies": { - "hosted-git-info": "^4.0.1", - "is-core-module": "^2.5.0", - "semver": "^7.3.4", - "validate-npm-package-license": "^3.0.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/normalize-package-data/node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/npm-normalize-package-bin": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", - "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/object.assign": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", - "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "has-symbols": "^1.0.3", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object.getownpropertydescriptors": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.6.tgz", - "integrity": "sha512-lq+61g26E/BgHv0ZTFgRvi7NMEPuAxLkFU7rukXjc/AlwH4Am5xXVnIXy3un1bg/JPbXHrixRkK1itUzzPiIjQ==", - "dependencies": { - "array.prototype.reduce": "^1.0.5", - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "es-abstract": "^1.21.2", - "safe-array-concat": "^1.0.0" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", - "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", - "dependencies": { - "yocto-queue": "^1.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", - "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", - "dependencies": { - "p-limit": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/path-exists": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", - "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" - }, - "node_modules/pkg-dir": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-7.0.0.tgz", - "integrity": "sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA==", - "dependencies": { - "find-up": "^6.3.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/protobufjs": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.6.tgz", - "integrity": "sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.5", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.1", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.1", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/pulumi": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/pulumi/-/pulumi-0.0.1.tgz", - "integrity": "sha512-Sow9mG2Yf3vRwQV5pux3EfZNvs7uoEjhYGTUWeGDTVrRT/6w3xybp8/a+SpDkH7dooHnGWDJW4QrSAbd+HvK5A==", - "deprecated": "To install Pulumi, use the @pulumi/pulumi package: 'npm -i @pulumi/pulumi'", - "dev": true - }, - "node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" - }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/read-package-json": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/read-package-json/-/read-package-json-2.1.2.tgz", - "integrity": "sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA==", - "dependencies": { - "glob": "^7.1.1", - "json-parse-even-better-errors": "^2.3.0", - "normalize-package-data": "^2.0.0", - "npm-normalize-package-bin": "^1.0.0" - } - }, - "node_modules/read-package-json/node_modules/hosted-git-info": { - "version": "2.8.9", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", - "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" - }, - "node_modules/read-package-json/node_modules/normalize-package-data": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", - "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", - "dependencies": { - "hosted-git-info": "^2.1.4", - "resolve": "^1.10.0", - "semver": "2 || 3 || 4 || 5", - "validate-npm-package-license": "^3.0.1" - } - }, - "node_modules/read-package-tree": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/read-package-tree/-/read-package-tree-5.3.1.tgz", - "integrity": "sha512-mLUDsD5JVtlZxjSlPPx1RETkNjjvQYuweKwNVt1Sn8kP5Jh44pvYuUHCp6xSVDZWbNxVxG5lyZJ921aJH61sTw==", - "deprecated": "The functionality that this package provided is now in @npmcli/arborist", - "dependencies": { - "read-package-json": "^2.0.0", - "readdir-scoped-modules": "^1.0.0", - "util-promisify": "^2.1.0" - } - }, - "node_modules/readdir-scoped-modules": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz", - "integrity": "sha512-asaikDeqAQg7JifRsZn1NJZXo9E+VwlyCfbkZhwyISinqk5zNS6266HS5kah6P0SaQKGF6SkNnZVHUzHFYxYDw==", - "deprecated": "This functionality has been moved to @npmcli/fs", - "dependencies": { - "debuglog": "^1.0.1", - "dezalgo": "^1.0.0", - "graceful-fs": "^4.1.2", - "once": "^1.3.0" - } - }, - "node_modules/regexp.prototype.flags": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", - "integrity": "sha512-0SutC3pNudRKgquxGoRGIz946MZVHqbNfPjBdxeOhBrdgDKlRoXmYLQN9xRbrR09ZXWeGAdPuif7egofn6v5LA==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.2.0", - "functions-have-names": "^1.2.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-5.2.0.tgz", - "integrity": "sha512-efCx3b+0Z69/LGJmm9Yvi4cqEdxnoGnxYxGxBghkkTTFeXRtTCmmhO0AnAfHz59k957uTSuy8WaHqOs8wbYUWg==", - "dependencies": { - "debug": "^4.1.1", - "module-details-from-path": "^1.0.3", - "resolve": "^1.22.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/resolve": { - "version": "1.22.4", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz", - "integrity": "sha512-PXNdCiPqDqeUou+w1C2eTQbNfxKSuMxqTCuvlmmMsk1NWHL5fRrhY6Pl0qEYYc6+QqGClco1Qj8XnjPego4wfg==", - "dependencies": { - "is-core-module": "^2.13.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/safe-array-concat": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.0.0.tgz", - "integrity": "sha512-9dVEFruWIsnie89yym+xWTAYASdpw3CJV7Li/6zBewGf9z2i1j31rP6jnY0pHEO4QZh6N0K11bFjWmdR8UGdPQ==", - "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.0", - "has-symbols": "^1.0.3", - "isarray": "^2.0.5" - }, - "engines": { - "node": ">=0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/safe-array-concat/node_modules/isarray": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" - }, - "node_modules/safe-regex-test": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", - "integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==", - "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.1.3", - "is-regex": "^1.1.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==", - "license": "ISC" - }, - "node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", - "bin": { - "semver": "bin/semver" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/shimmer": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", - "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" - }, - "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", - "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/spdx-correct": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", - "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", - "dependencies": { - "spdx-expression-parse": "^3.0.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-exceptions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", - "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==" - }, - "node_modules/spdx-expression-parse": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", - "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", - "dependencies": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "node_modules/spdx-license-ids": { - "version": "3.0.13", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.13.tgz", - "integrity": "sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w==" - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string.prototype.trim": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz", - "integrity": "sha512-p6TmeT1T3411M8Cgg9wBTMRtY2q9+PNy9EV1i2lIXUN/btt763oIfxwN3RR8VU6wHX8j/1CFy0L+YuThm6bgOg==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/string.prototype.trimend": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz", - "integrity": "sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/string.prototype.trimstart": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz", - "integrity": "sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "engines": { - "node": ">=6" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/ts-node": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-7.0.1.tgz", - "integrity": "sha512-BVwVbPJRspzNh2yfslyT1PSbl5uIk03EZlb493RKHN4qej/D06n1cEhjlOJG69oFsE7OT8XjpTUcYf6pKTLMhw==", - "dependencies": { - "arrify": "^1.0.0", - "buffer-from": "^1.1.0", - "diff": "^3.1.0", - "make-error": "^1.1.1", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "source-map-support": "^0.5.6", - "yn": "^2.0.0" - }, - "bin": { - "ts-node": "dist/bin.js" - }, - "engines": { - "node": ">=4.2.0" - } - }, - "node_modules/typed-array-buffer": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz", - "integrity": "sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw==", - "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.1", - "is-typed-array": "^1.1.10" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/typed-array-byte-length": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.0.tgz", - "integrity": "sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA==", - "dependencies": { - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "has-proto": "^1.0.1", - "is-typed-array": "^1.1.10" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/typed-array-byte-offset": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.0.tgz", - "integrity": "sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg==", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "has-proto": "^1.0.1", - "is-typed-array": "^1.1.10" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/typed-array-length": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz", - "integrity": "sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==", - "dependencies": { - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "is-typed-array": "^1.1.9" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/typescript": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", - "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, - "node_modules/unbox-primitive": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", - "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", - "dependencies": { - "call-bind": "^1.0.2", - "has-bigints": "^1.0.2", - "has-symbols": "^1.0.3", - "which-boxed-primitive": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/upath": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", - "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", - "engines": { - "node": ">=4", - "yarn": "*" - } - }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, - "node_modules/util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, - "node_modules/util-promisify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/util-promisify/-/util-promisify-2.1.0.tgz", - "integrity": "sha512-K+5eQPYs14b3+E+hmE2J6gCZ4JmMl9DbYS6BeP2CHq6WMuNxErxf5B/n0fz85L8zUuoO6rIzNNmIQDu/j+1OcA==", - "dependencies": { - "object.getownpropertydescriptors": "^2.0.3" - } - }, - "node_modules/uuid": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", - "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/validate-npm-package-license": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", - "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", - "dependencies": { - "spdx-correct": "^3.0.0", - "spdx-expression-parse": "^3.0.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/which-boxed-primitive": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", - "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", - "dependencies": { - "is-bigint": "^1.0.1", - "is-boolean-object": "^1.1.0", - "is-number-object": "^1.0.4", - "is-string": "^1.0.5", - "is-symbol": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" - }, - "node_modules/xml2js": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", - "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", - "license": "MIT", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yn": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/yn/-/yn-2.0.0.tgz", - "integrity": "sha512-uTv8J/wiWTgUTg+9vLTi//leUl5vDQS6uii/emeTb2ssY7vl6QWf2fFbIIGjnhjvbdKlU0ed7QPgY1htTC86jQ==", - "engines": { - "node": ">=4" - } - }, - "node_modules/yocto-queue": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", - "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } -} diff --git a/chat-sdk/.gitignore b/chat-sdk/.gitignore new file mode 100644 index 0000000000..1eae0cf670 --- /dev/null +++ b/chat-sdk/.gitignore @@ -0,0 +1,2 @@ +dist/ +node_modules/ diff --git a/chat-sdk/README.md b/chat-sdk/README.md new file mode 100644 index 0000000000..1d09b68265 --- /dev/null +++ b/chat-sdk/README.md @@ -0,0 +1,317 @@ +# windmill-chat + +Build a chat interface on a Windmill flow deployed in **chat mode**, from any frontend +or from a Windmill raw app. The library is headless: it runs the flow, follows the +answer as it streams, keeps the conversation history, and hands you state to render. + +``` +npm install windmill-chat +``` + +No runtime dependencies. Optional peers: `react` for `windmill-chat/react`, `ai` for +`windmill-chat/ai-sdk`, `@assistant-ui/react` for `windmill-chat/assistant-ui`. + +| You build the UI with | Import | You get | +|---|---|---| +| Vercel AI SDK `useChat`, AI Elements | `windmill-chat/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | +| assistant-ui | `windmill-chat/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | +| Your own components | `windmill-chat/react` or `windmill-chat` | a hook / a store with messages, status and actions | + +## The flow + +Any deployed flow with **Chat mode** enabled in its settings works. Windmill passes the +message as the `user_message` input and threads the conversation through `memory_id`, +so an AI agent step remembers earlier turns. The answer is: + +- what the last step streams, when it is an AI agent step; +- otherwise the flow's result: its `windmill_chat_answer` field when it has one, a + string as is, anything else as JSON. + +## Vercel AI SDK (`useChat`, AI Elements) + +```tsx +import { useChat } from '@ai-sdk/react' +import { createWindmillChatTransport } from 'windmill-chat/ai-sdk' + +const transport = createWindmillChatTransport({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) +}) + +export function Support() { + const { messages, status, sendMessage, stop } = useChat({ id: conversationId, transport }) + // render `messages[i].parts`: text, reasoning and dynamic-tool parts, as with any AI SDK backend +} +``` + +The chat `id` is the conversation: reuse it to continue one, and pass a UUID when you +also read server history, so it matches what `flow_conversations` stores (any other id +maps to a fixed UUID). `sendMessage(msg, { body })` sends extra flow inputs. Tool calls +arrive as `dynamic-tool` parts (`input-available → output-available | output-error`), +which AI Elements' `` renders as is. A failed flow surfaces as `error`. +`regenerate()` runs the flow again with the same message: a new turn on the server, +not a replacement of the previous answer. + +The transport also carries the history helpers: `transport.loadMessages(id)` returns +`UIMessage`s for `useChat({ messages })` or `setMessages`, `transport.listConversations()` +and `transport.deleteConversation(id)`. A loaded user message lists the files it carried +in `metadata.attachments`; `WindmillChatApi.attachmentUrl` gives each one's download URL. +Sending attachments is not supported: `sendMessage` with `files` is refused with an +explanatory error. + +## assistant-ui + +```tsx +import { AssistantRuntimeProvider } from '@assistant-ui/react' +import { useWindmillRuntime } from 'windmill-chat/assistant-ui' + +export function Support() { + const runtime = useWindmillRuntime({ baseUrl, workspace, flowPath, token }) + return ( + + {/* your assistant-ui components, thread list included */} + + ) +} +``` + +Conversations are threads: `ThreadListPrimitive` switches, creates and deletes them. +Tool calls render through your `tools` components (`MessagePrimitive.Parts`), reasoning +through `Reasoning`. It takes the same options as `useWindmillChat` below. + +## React + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +export function Support() { + const chat = useWindmillChat({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) + }) + const [draft, setDraft] = useState('') + + return ( +
+ {chat.messages.map((m) => ( +

+ {m.content} +

+ ))} +
{ + e.preventDefault() + chat.sendMessage(draft) + setDraft('') + }} + > + setDraft(e.target.value)} /> + + {chat.status === 'streaming' && ( + + )} + +
+ ) +} +``` + +The hook returns the [state](#state) plus the chat's methods. It recreates the chat +(fresh state, old one destroyed) when `flowPath`, `baseUrl`, `workspace`, `history`, +`storageKey` or the credential change: a different token string, or a switch between +no token, a string and a function; and when a `run` callback appears or goes away. +A token function is called through a ref, so passing a new closure on every render +is fine and never resets the chat, and so are `run` and the callbacks; when users +sign in and out behind a token function, change `storageKey` (their id) so local +history and state start over with them. + +## Raw apps + +Inside a Windmill raw app nothing needs configuring: the chat runs as the viewer, +against the Windmill the app is served from. + +```tsx +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +``` + +- **Unsandboxed app** (the default): the viewer's session is used. Viewers need + permission to run the flow. +- **Sandboxed app**: declare `jobs:run` in the app's frontend SDK scopes, and + `flow_conversations:write` for server-side history. The viewer consents once and + the app receives a token restricted to those scopes. +- **`wmill app dev`**: there is no viewer session on the dev server, so pass + `baseUrl`, `workspace` and `token` explicitly during development. + +## Any framework + +`createChat` returns a store: `subscribe` calls the listener immediately and on every +change, and returns the unsubscribe function. That is the Svelte store contract, so +`$chat` works as is; other frameworks wrap it in a few lines. + +```ts +import { createChat } from 'windmill-chat' + +const chat = createChat({ baseUrl, workspace, flowPath, token }) +chat.subscribe((state) => render(state)) +await chat.sendMessage('Hello') +``` + +```svelte + + +{#each $chat.messages as m (m.id)} +

{m.content}

+{/each} + +``` + +## Options + +| Option | | +|---|---| +| `flowPath` | Path of the deployed flow, e.g. `f/support/assistant`. Required. | +| `baseUrl` | The Windmill origin. Detected inside a raw app. | +| `workspace` | Detected inside a raw app. | +| `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | +| `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. | +| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones, and `{ attachments, attachmentsInput }` files, see [Attachments](#attachments). | +| `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. | +| `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. | +| `pageSize` | Messages and conversations per page of server history. Default 50. | +| `pollDelayMs` | How often, in ms, the server polls a running turn for the stream (Enterprise; 50 at the fastest, other servers ignore it). Unset, the server relaxes from 100 ms to 3 s over a long turn; set it when tokens must keep flowing at that pace. | +| `onFinish`, `onError` | Called when a turn has its answer, or could not run at all. | +| `run` | Runs the flow for a turn yourself and returns the job id, instead of the deployed flow at `flowPath` (Windmill's editor chats with an undeployed flow through a preview run this way). Pass `memory_id` = the conversation id. | + +## State + +```ts +interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: 'idle' | 'submitted' | 'streaming' | 'error' + error: Error | undefined + conversations: Conversation[] + history: 'server' | 'local' | 'none' + loadingMessages: boolean + hasMoreMessages: boolean +} + +interface ChatMessage { + id: string + role: 'user' | 'assistant' | 'tool' | 'system' + content: string + reasoning?: string // the model's reasoning summary, when streamed + tool?: { callId?: string; name: string; arguments?: string; result?: string; status: 'running' | 'success' | 'error' } + success: boolean // false for a failed flow or tool + pending: boolean // still streaming, or not yet confirmed by the server + createdAt: string + jobId?: string + stepName?: string + serverId?: string // the persisted row; `id` itself never changes, so list keys are stable +} +``` + +A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arriving) → +`idle`. Tool calls appear as `tool` messages whose `status` moves from `running` to +`success` or `error`. A flow that fails still completes the turn: its error is the +answer, an `assistant` message with `success: false`. `status: 'error'` (with `error` +set) means the turn could not run or be followed at all, such as a refused request. + +Methods: `sendMessage(text, { inputs?, attachments?, attachmentsInput? })`, `stop()`, +`newConversation()`, `selectConversation(id)`, `loadConversations({ page?, perPage?, kind? })`, +`deleteConversation(id)`, `renameConversation(id, title)`, `loadOlderMessages()`, +`destroy()`. `kind` lists the flow editor's test chats (`'test'`), the deployed flow's +own (`'deployed'`, the server's default) or both (`'all'`); each `Conversation` carries +`isTest`. A rename keeps the conversation's place in the list. Switching conversations +stops following the current answer; the flow keeps running and, with server history, +its answer is there when you come back. + +## Attachments + +A flow whose AI agent step reads `user_attachments` from an `s3object[]` (or a single +`s3object`) flow input takes files with a message: + +```ts +await chat.sendMessage('What does this contract say?', { + attachments: [{ name: file.name, data: file }], // a Blob/File, or a `data:` URL + attachmentsInput: { name: 'files', multiple: true } +}) +``` + +Each file is uploaded to the workspace's object storage under +`windmill_uploads/chat///` and handed to the input as `{ s3, filename }` +objects (the object for a single-file input). Once the uploads return, the pending user +message lists them in `attachments`, as `{ input, s3, filename }` references. The name's +extension is corrected to the file's media type for PNG, JPEG and PDF, because the worker +reads the type off the key. +Files need message text to go with them. A failed upload rejects `sendMessage` before any +run starts, and `stop()` during the upload aborts it; both leave the transcript as it was. +The chat never deletes uploads, so files of a send that did not run stay in storage. The +workspace needs object storage set up. With Enterprise advanced storage permissions, the +user needs read and write on `windmill_uploads/*`, which the default rules grant. The upload goes through +`job_helpers`, so a restricted token needs `job_helpers:write`; a sandboxed raw app cannot +request that scope today, so attachments are not available there yet. + +## History + +Windmill stores every conversation of a chat-mode flow, and each Windmill user sees +only their own. `history: 'server'` reads that store: `loadConversations()` lists +them, `selectConversation(id)` loads one, `loadOlderMessages()` pages back. Every +message rendered from the server carries its `jobId` and `stepName`. + +That store is keyed by the **Windmill user**, so it fits a viewer session or a token +issued per user. With one token shared by every visitor of a site, all visitors would +see each other's conversations. For that setup use `history: 'local'` (the default +with an explicit `token`): the conversation list and messages stay in the browser's +`localStorage`, per Windmill instance, workspace and flow. `'none'` keeps nothing +beyond the page. + +When the default `'server'` mode turns out unreadable (a token or sandboxed app +without `flow_conversations` scopes), the chat switches itself to `'local'` and +`state.history` says so. Passing `history` explicitly disables that fallback. + +## Tokens + +Anything a browser holds can be read by its user, so give a chat token exactly what +the chat needs: + +| Setup | Scopes | +|---|---| +| Public site, one token for everyone | `jobs:run:flows:f/support/assistant`, and `history: 'local'`. The token can run that one flow and follow its jobs, nothing else. | +| Per-user tokens minted by your backend | The above plus `flow_conversations:write`, with `history: 'server'` (an explicit token defaults to local history). Return them from an endpoint and pass `token: () => fetch(...)`. | +| A Windmill user in the browser (raw app, embedded Windmill) | No token: the session is used. | + +The token's user must be allowed to run the flow. `stop()` closes the stream in any +case; cancelling the run on the server as well needs `jobs:write`, which also lets the +token read every job its user can see, so leave it out unless that matters. + +Anyone holding the token can run the flow with inputs of their choosing, so a flow +exposed this way should treat `user_message` and the other inputs as untrusted. + +## Lower level + +`WindmillChatApi` wraps the endpoints (`runFlow`, `streamJob`, `listConversations`, +`listMessages`, `deleteConversation`, `cancelJob`), `followJob` follows a run to +completion across the server's stream timeouts, `parseStreamEvents` decodes the AI +agent stream, `extractChatAnswer` turns a flow result into the text a chat shows, and +`conversationIdFor` maps any chat id to its conversation UUID. They are exported for +custom integrations. + +## For AI coding agents + +When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode +on. Pick the entry point from the table at the top (`useChat` → `windmill-chat/ai-sdk`, +assistant-ui → `windmill-chat/assistant-ui`, otherwise `windmill-chat/react`). Inside a +Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a +`token`; for a public page use a token scoped to `jobs:run:flows:` and leave +`history` at its default. Render `role`, `content`, `pending`, `success` and +`tool.status`; never build the SSE handling yourself. diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json new file mode 100644 index 0000000000..e657c139d7 --- /dev/null +++ b/chat-sdk/package-lock.json @@ -0,0 +1,3433 @@ +{ + "name": "windmill-chat", + "version": "1.816.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "windmill-chat", + "version": "1.816.0", + "license": "Apache-2.0", + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.80", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.80.tgz", + "integrity": "sha512-6t+07o8lSthpKf64Xb1qHWR2bWvJ3Fd2oFvS9fQc45p31bi2OUqan246e/ojAmZpWCiMPjKyQ4TBr4MYytnTiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/mcp": { + "version": "2.0.49", + "resolved": "https://registry.npmjs.org/@ai-sdk/mcp/-/mcp-2.0.49.tgz", + "integrity": "sha512-dD8//65te2C5B4UzPNHGLR7CStR5IvY7o7kXz0YYtPQs+aCdXc7Jb3fuvZcovShfP1KGvPIMTfXhp6Uz+UghJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "cross-spawn": "^7.0.6", + "pkce-challenge": "^5.0.1" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.14.tgz", + "integrity": "sha512-yukP2tbcQQErG5gLCMBGvpvb/rM3D3KlTKG6eKKOdNHLHqtNNDEeBxdYrY/JL+O76B2ig5dXY19H/f1HFSvRiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.40.tgz", + "integrity": "sha512-zsXPwSAQ9mRJ2hvyITaLOYUyuGrBmzFhJXOg4mllGmla1PfNxZcm4GwqMiV2xQaDgcgBMdUGxXkp9xekZZNIkg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "4.0.102", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-4.0.102.tgz", + "integrity": "sha512-KOTRsaVUr6QeisktVqm57KfBMqBpxK9f2ay+defGnYsfvXpFE277zg107Q27LSpiTsDpmk2GHMqNWdU9bKaJSw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/mcp": "2.0.49", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "ai": "7.0.99", + "swr": "^2.4.1", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "react": "^18 || ~19.0.1 || ~19.1.2 || ^19.2.1" + } + }, + "node_modules/@assistant-ui/core": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@assistant-ui/core/-/core-0.3.18.tgz", + "integrity": "sha512-WR5/uqZuI6FNWogIe5EKmP7gdLhQhX38QqaAMAfH9DEPVzw40ON4sZC9vD47rpBj6yrXhHIsbt6jAGUKH4mASQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42", + "nanoid": "^6.0.1" + }, + "peerDependencies": { + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "assistant-cloud": "^0.2.0", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "assistant-cloud": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/react": { + "version": "0.15.19", + "resolved": "https://registry.npmjs.org/@assistant-ui/react/-/react-0.15.19.tgz", + "integrity": "sha512-+mEXA/ibBSoodj3LFUzRAdYZ+HvlrIg8uexPVjcY3ssksG1zHx8E9TpuwCDxg2h3XrYU8c/UZj/uFlI5tTWMoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@assistant-ui/core": "^0.3.18", + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "assistant-cloud": "^0.2.0", + "assistant-stream": "^0.3.42", + "radix-ui": "^1.6.7", + "react-textarea-autosize": "^8.5.9", + "safe-content-frame": "^0.0.30", + "zod": "^4.5.4", + "zustand": "^5.0.15" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/store": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@assistant-ui/store/-/store-0.3.13.tgz", + "integrity": "sha512-4u5YAMfjgr+jpJUWZ5f1uqXQuxvIiR8SRg1WWlFbIknk5S1Wli4Jp/nTQsOeXsIw0d7Zdb6FKiQw31gqs5z/Ew==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/tap": { + "version": "0.9.17", + "resolved": "https://registry.npmjs.org/@assistant-ui/tap/-/tap-0.9.17.tgz", + "integrity": "sha512-z53TiHiM3ai8XQ5B60Lg8IKCWM0XRf1Sz1jTQtqiuAATzVd6zeFedSHpxq4i5P0zi5uYVdN/dkGEpVTOyPc3ow==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "dev": true, + "license": "MIT" + }, + "node_modules/@happy-dom/global-registrator": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/@happy-dom/global-registrator/-/global-registrator-20.14.5.tgz", + "integrity": "sha512-B05ID9DhSwLs6mlm1fzlkAtTIvB3duCvjJjfr19LBrlTK7VZtRjDqoTRIVv13GuYuNdhByu8LSZgThwV3Rkj7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "happy-dom": "^20.14.5" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.149.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@quansync/fs": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/@radix-ui/number": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", + "integrity": "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.7.tgz", + "integrity": "sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.1.15.tgz", + "integrity": "sha512-WTQwcAvQf5sOcuUyi90lKPbhwcvQ+j55cjrSmeaN+L2vKU3DooOvlKw2MDeiJ5IkV5N905KW0/fGojKOBhD11A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-accordion": { + "version": "1.2.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.20.tgz", + "integrity": "sha512-jDhG9FvAEnlhnjrsINbNXcUa4G+L1KqSkJSunkbKEzFRcAb52jvM0PjPxPRvhe1HNc5F5yc0yzzWeeqlH4yBIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.1.23.tgz", + "integrity": "sha512-VAYOiQRqj3GPpYJE0I9J+X8Ip05cyVlNdKOFeiGS2Ou1HHGfpl0BxOyZm6nmVDyU+W+NF3/XLzmjHmVGydhwgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.15.tgz", + "integrity": "sha512-v4zggRcjadnI+ClKDuijlQEW4tw3NoaeHc/PwpKnLoLLKNUG4InLegkstooLcRIUWCs+8L22dGURCVuFfOKfnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.1.15.tgz", + "integrity": "sha512-fy+dyVR+90nelK8rqIznFlxzx7uPcGbhxH8Nfr2bHb4UfSe+e3hklOC0luK0hDwVwnRX7xTRySpsrQVeW+/oNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-avatar": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.2.6.tgz", + "integrity": "sha512-4ULOTJ/mqy2hT9GlWa/MFHxHSvH3nJzHnZM1waNsc5Bonv7i70aNenghXmD97S6OJ81ekXONGGt4nT1r0PfEdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-checkbox": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.3.11.tgz", + "integrity": "sha512-Gnptr9pDDQxD3hgq2dtPbtrp/c2qH1mBwIzw3X/ivrMb2e1t0jMTi606fVEqFPaQR1ggXIVQWKj3P2WW9v7zGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collapsible": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.20.tgz", + "integrity": "sha512-mcGesGplBnzN2sbvJETzpCNfSMyPnb29q1GRLU+Ib7bJrpIG2ywmRoh2V5VbA2uNvKikKUlVbAPks7JDjz4A8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.15.tgz", + "integrity": "sha512-9W+B9NPF0NaaPh/1NJd3+KqsnlLqU9H7T2rvww+fp+T/evVXdNAyYcnfRQZFOjkR1ajQp3yORlqnI8soawLvNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.2.tgz", + "integrity": "sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.3.7.tgz", + "integrity": "sha512-CtXP35dxaB5T3zXSd+E3uHe/QpXcpYnZmxp6OaIbfthtfW4wyb77M23BG+bwIJDtsMwEP/YssdsmNyZu7jhWew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.23.tgz", + "integrity": "sha512-Ksw4WeROkO4rC9k/onilX/Ao2Cr1ku1unMNH+XSCcP4jSXYu7HDsg9n4ojMjVb22XpYjAQ9qfrFlVbru1vXDUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.4.tgz", + "integrity": "sha512-5pzg4FGQNpExhnhT2zlrP1wZFaYCd1K0nYWoFAdcYoYK868IEigqMX3B3f8yIoRlAhAeDWciLI6ZdCKHF9P4Vg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.19.tgz", + "integrity": "sha512-8g4pfOL9HoKKLWGiypT+dphVqjFfmcXO5GBnhsG6zI+lxAx/8feQpr+1LSN8Re3hiZ+XkLNS4O9ztK11/LzQ6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.24.tgz", + "integrity": "sha512-geq8l2rJkxvkXsT9RMgtUE3P8pITFpTsvYpbySi1IH4fZEABD/Gp85myayFgxk0ktljGMJnCbeFkyTusvSvv7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.6.tgz", + "integrity": "sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.16.tgz", + "integrity": "sha512-wmRZ2WWLvmt6KHy2rNPOdPUjwq5xOHY02+m+udwJTn0aNIox/rkskAvJTyTLGhPK6KgrUjlJUJpgmx/+wFiFIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-form": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-form/-/react-form-0.1.16.tgz", + "integrity": "sha512-Q4TLEn2A7TAypxwmd6R9EwrlXDvkfYSDMrq9/887AXAGh+G1rH+kYJKSTv+Si9Y0JPKTwKYv6PviAJosysNimA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.1.23.tgz", + "integrity": "sha512-H8qONfZd3ltrU3+jHCIgITbWo6e1iTKvP9DHdrvYbX48ooRM5FjEDTn16AMwdfuOGkWdZEhpl3PLL/Wk/AnHDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.15.tgz", + "integrity": "sha512-o/rdYEwZTTo5tjknnPeyQFU45kUC4i/XyeDPP+HGyi6XqpOP6Zf5Ya5vh/Yfe9Id5JiuWnnAx2XqIeD3UYZt0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.24.tgz", + "integrity": "sha512-uW7RVuU6Lp/ZtfeY4b3kL32zccgEWvPv1+cf17ubYzHa9cL8AHokmk36cG/XEiH/smbQvumnieXX9j/e9RqJWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menubar": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menubar/-/react-menubar-1.1.24.tgz", + "integrity": "sha512-eeVs0vf7cuqXaM0qLQCPcufImiJNVBXdJDLu7ZGYl2732UH23Qat/foNGrr6vYV3/DdTsBqASoggUFgH14OcZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.22.tgz", + "integrity": "sha512-ou7iLEJ+yrhQndkkA4U21XIdS/CS45F4iXIkTZcb6/Ne9EMsOuDudVmCwmDnfFZZ+y1FZqXRNSIgBy+YMvZVZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-one-time-password-field": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-one-time-password-field/-/react-one-time-password-field-0.1.16.tgz", + "integrity": "sha512-Tj9P6ntAJEw52oq/F0AGknXR4XncxEt7XU47O3xJQOiWfLzEy3d9gtgKfvjSzGxzHkfL+VzvxGu2KTFsloJqXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-password-toggle-field": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-password-toggle-field/-/react-password-toggle-field-0.1.11.tgz", + "integrity": "sha512-4gvFnmDXu3dgj21CqsufzIameRvlRd4SBqaWhcrlrNhRo0Y5i/49AmRJYe1fdAM3G2VNBbmin4b0D6cdQocwgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.23.tgz", + "integrity": "sha512-mw58MrBlyHWFisTOYignD0vf/3gdcgAR+9of1s9G/38CbFiUwH1nCDkc0AUM9IrXFgN5Ue8n45j9WCgyM1sbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.7.tgz", + "integrity": "sha512-UsJrrd7w4wuKKTdvd/DNERVlwSlUcyXzjhyDwBk+3aPOsCjOY6ZSbxuw8E6lZTjjfP8Cpd0J8VVkrYUWyGYXyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-rect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.17.tgz", + "integrity": "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.10.tgz", + "integrity": "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.10.tgz", + "integrity": "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.16.tgz", + "integrity": "sha512-5XnomAsoZZCY+KNTxbIghpGqPruZvKFNlvcAljVAOdDRDsH4/OZQxhtwo5wdtoDM5R6MhJBb2sPnDuRFep3lzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.4.7.tgz", + "integrity": "sha512-cgYFEkntCxppHZgtSZ+7vh0wbZQ+IC7PPMw8DSnRG27B6kDd32/Zw0OJt7dGDigCoprMuWHjg2PvUn3PYvPFoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.19.tgz", + "integrity": "sha512-V9jI6hDjT7l3jsCQD9bLNvDLM3tH/gdbOTp7Tefp3hbbgCGQoK7tUvrWiRlcoBHIZ809ElXwNQwVo0B98LuTXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.18.tgz", + "integrity": "sha512-Zn5Cd171wxsO3Dfg8HaW6RifTb9CYTKQJHs/G4+LN1GfmJpaQMZQyQxMprVPHpaz7QY4l9BxK2JwQuzHsXC8nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.7.tgz", + "integrity": "sha512-WFGImkmbzcfxeIwq/+4HvRN0pizBwbwQUED4I13ezQsDdfl38ZntN6TmR8XaSzPBqoCToe8rF75j6NPNDSzhbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.15.tgz", + "integrity": "sha512-jOLO4lssEzWpoDu7G+Ze4VjwMRUBt291pnZD0gmalREZipnTX3wadQo7Fy48GCTfe14/YRN6rw/rOJqrE85Wxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.7.tgz", + "integrity": "sha512-mTSLf1GC/C0moWjTbvCM6Qn/gBjvlFt1azuWF2v7MN5C3Zq2U2J2lN3ZEYkpujuOU5Ro7A28wkviSxaKnG0BYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.3.3.tgz", + "integrity": "sha512-qx7oqnYbxnK9kYI9m317qmFmEgo6ywqWvbTogdj7cL9p3/yx4M48p7Rnw5z3H890cL/ow/EeWJsuTykeZVXP5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.3.7.tgz", + "integrity": "sha512-48tB/4dn2UVLBCYhTu9AuR63IHl73l/qLbLgxd86noTUor4/K4LFDAcYjK+isP5313qxaFpjPVogE7+Y0/V3Kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.21.tgz", + "integrity": "sha512-UKxJlZid7FVtsk/WTxj4i4uSEgj2Au+KBbS7SQyTlzMhhn+86Cz3tISZdTa87bfEfcuvZezf2ZsxD4xuEKtkog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.23.tgz", + "integrity": "sha512-ofhyAsYaocRGOs/n0XWdUOSVzEAG6BfrMVM8z0c0kLEWY38w/0WuMFPTJP/HVaZPYkMvHZoKIIhNcjbTCBILPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.18.tgz", + "integrity": "sha512-7lonPlKfSacd20GlOBx2ltuVKz9oqWYZz+oMQyOltw6t1y2nyftj2ZmwwUHYn49kqfDWcp8dNZm5NgV+5Z+mug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.1.19.tgz", + "integrity": "sha512-OtnwuSVjd1Ofi+AdnvhsjQdyuhCDwYs1w9RyB5BN/OavXOVQo42SYqQjwUnbPnaiPFBpQ9aX70dWeee+v2oBLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.1.19.tgz", + "integrity": "sha512-Ph0IvtYw4VB12ZnZg+YtrGs8yJQsnizwo/zu0R4Y/nWugtJzA7Pg1eWeuDR9+LSqn+xjamss+UOSOJJJ4gx8jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-toggle-group": "1.1.19" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.16.tgz", + "integrity": "sha512-6EamKFRRnlpdadndbZ6LMwycfwkwPte1B42hs6QA0gYhjaOKqW4PZ4pjaW9UrlDX5eVt/OjncE7BFTPL5nmZhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.4.tgz", + "integrity": "sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.6.tgz", + "integrity": "sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.5.tgz", + "integrity": "sha512-ge3ipobwSXTj4JyVtswQ7qZj0ZHdtbGuOno/LrgAAeSxtsJ6Vs4Gz5IkPH2bmqpjcLUFoqGhA/mueuIf63UXlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-is-hydrated": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-is-hydrated/-/react-use-is-hydrated-0.1.3.tgz", + "integrity": "sha512-umO/aJ+82CpOnhDZUTbILCQf7kU/g0iv+oGs/Q8jw7IkhWBzaEP4sA268PhFAJTFetbwp3ICc6ktpI4TqtxcIw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.4.tgz", + "integrity": "sha512-XoSLhbRbqxFtgJoi2fNHA3C6pDlY34x508vUpUGoFZfvePfHXHbE1lC4FYFMnJWgiCRroSTw6fOsXQoVS9RwZg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.4.tgz", + "integrity": "sha512-cSOCh6JlkmfjLyNcLiu2nB4v+nm+dkZ+Q5KHWk/soo4U7ZLiEQFKHK9/YmtBHjfCEaU43IBKQOc4/uJmCaiCTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.4.tgz", + "integrity": "sha512-D3anSY15EJoxrihpsXI6SMrmmonnQtR2ni7arO+Lfdg3O95b9hNXxONk8jA5C8ANdF/h5HMAxejgs8PWJ6rlhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.11.tgz", + "integrity": "sha512-NFS86RYYZb4/exihaESBGOpMJFz8MGLAfu3mOBSGByVnVPC9JPASfYubxd/8KbkQK0sYAv8lVQDEQukDX/qXvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.3.tgz", + "integrity": "sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.8.tgz", + "integrity": "sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.8.tgz", + "integrity": "sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.8.tgz", + "integrity": "sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.8.tgz", + "integrity": "sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.8.tgz", + "integrity": "sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.8.tgz", + "integrity": "sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.8.tgz", + "integrity": "sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.8.tgz", + "integrity": "sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.8.tgz", + "integrity": "sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.8.tgz", + "integrity": "sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.8.tgz", + "integrity": "sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.8.tgz", + "integrity": "sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.8.tgz", + "integrity": "sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/bun": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.4.2" + } + }, + "node_modules/@types/node": { + "version": "26.5.1", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@types/whatwg-mimetype": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-3.0.2.tgz", + "integrity": "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.99", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.99.tgz", + "integrity": "sha512-Ov+3j/nSajaVH5hO8C94wN9wisG5tAJ8HWsLV9d/+nlzrRGUh3oYO3Kp1fRyUNWjAWLSpGLHLPaHCdfxb307Vg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.80", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ansis": { + "version": "4.4.0", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/assistant-cloud": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/assistant-cloud/-/assistant-cloud-0.2.0.tgz", + "integrity": "sha512-LMvPaufIfZ0dpByMsNeuPFY6UeKQ1oiSBktUJuhjL/2eTZLSzzjIWumiLALlpdvUZbbJCRHy8j0ZgzlZJJlEfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.200.0", + "@opentelemetry/sdk-trace-base": "^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/assistant-stream": { + "version": "0.3.42", + "resolved": "https://registry.npmjs.org/assistant-stream/-/assistant-stream-0.3.42.tgz", + "integrity": "sha512-5dQxc7XX92LuJ8wuHaOi/vkHitz2DARABzoE364O7o4jiePWalDpDpqU9bN+bXTvnSiRrpesy6t3Z3oDqRkdQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "nanoid": "^6.0.1", + "secure-json-parse": "^4.1.0" + }, + "peerDependencies": { + "ioredis": "^5.10.1 || ^6.0.0", + "redis": "^5.12.1" + }, + "peerDependenciesMeta": { + "ioredis": { + "optional": true + }, + "redis": { + "optional": true + } + } + }, + "node_modules/ast-kit": { + "version": "2.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "pathe": "^2.0.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/birpc": { + "version": "2.9.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/buffer-image-size": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/buffer-image-size/-/buffer-image-size-0.6.4.tgz", + "integrity": "sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/bun-types": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/defu": { + "version": "6.1.7", + "dev": true, + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/diff": { + "version": "8.0.4", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dts-resolver": { + "version": "2.1.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "oxc-resolver": ">=11.0.0" + }, + "peerDependenciesMeta": { + "oxc-resolver": { + "optional": true + } + } + }, + "node_modules/empathic": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.3", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/happy-dom": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-20.14.5.tgz", + "integrity": "sha512-x/RzkpWO40bTjIoT30iQtt64FLLmH/iRcUCN2X//bLx7H3ifkdfPXyqsro/OYtqzIAhiLMMA7mmiOR9C3NOKjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "@types/whatwg-mimetype": "^3.0.2", + "@types/ws": "^8.18.1", + "buffer-image-size": "^0.6.4", + "entities": "^7.0.1", + "whatwg-mimetype": "^3.0.0", + "ws": "^8.21.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/hookable": { + "version": "5.5.3", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true, + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/ms": { + "version": "2.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-6.0.1.tgz", + "integrity": "sha512-3wVS3i51pE2pi1k5FFL/95BGfVS0kSsvDVuGXHOtxox/TywUmtgq+3qiTOTbs9J7KfHaXPiN171k/A6dBnaXFw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^22 || ^24 || >=26" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/quansync": { + "version": "1.0.0", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/radix-ui": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.6.7.tgz", + "integrity": "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-accessible-icon": "1.1.15", + "@radix-ui/react-accordion": "1.2.20", + "@radix-ui/react-alert-dialog": "1.1.23", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-aspect-ratio": "1.1.15", + "@radix-ui/react-avatar": "1.2.6", + "@radix-ui/react-checkbox": "1.3.11", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-context-menu": "2.3.7", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-dropdown-menu": "2.1.24", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-form": "0.1.16", + "@radix-ui/react-hover-card": "1.1.23", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-menubar": "1.1.24", + "@radix-ui/react-navigation-menu": "1.2.22", + "@radix-ui/react-one-time-password-field": "0.1.16", + "@radix-ui/react-password-toggle-field": "0.1.11", + "@radix-ui/react-popover": "1.1.23", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-progress": "1.1.16", + "@radix-ui/react-radio-group": "1.4.7", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-scroll-area": "1.2.18", + "@radix-ui/react-select": "2.3.7", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-slider": "1.4.7", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-switch": "1.3.7", + "@radix-ui/react-tabs": "1.1.21", + "@radix-ui/react-toast": "1.2.23", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-toggle-group": "1.1.19", + "@radix-ui/react-toolbar": "1.1.19", + "@radix-ui/react-tooltip": "1.2.16", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-escape-keydown": "1.1.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-textarea-autosize": { + "version": "8.5.9", + "resolved": "https://registry.npmjs.org/react-textarea-autosize/-/react-textarea-autosize-8.5.9.tgz", + "integrity": "sha512-U1DGlIQN5AwgjTyOEnI1oCcMuEr1pv1qOtklB2l4nyMGbHzWrI0eFsYK0zos2YWqAolJyG0IWJaqWmWj5ETh0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.13", + "use-composed-ref": "^1.3.0", + "use-latest": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/rolldown": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.149.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.8", + "@rolldown/binding-android-arm64": "1.2.8", + "@rolldown/binding-darwin-arm64": "1.2.8", + "@rolldown/binding-darwin-x64": "1.2.8", + "@rolldown/binding-freebsd-x64": "1.2.8", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.8", + "@rolldown/binding-linux-arm64-gnu": "1.2.8", + "@rolldown/binding-linux-arm64-musl": "1.2.8", + "@rolldown/binding-linux-ppc64-gnu": "1.2.8", + "@rolldown/binding-linux-s390x-gnu": "1.2.8", + "@rolldown/binding-linux-x64-gnu": "1.2.8", + "@rolldown/binding-linux-x64-musl": "1.2.8", + "@rolldown/binding-openharmony-arm64": "1.2.8", + "@rolldown/binding-win32-arm64-msvc": "1.2.8", + "@rolldown/binding-win32-x64-msvc": "1.2.8" + } + }, + "node_modules/rolldown-plugin-dts": { + "version": "0.13.14", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.1", + "ast-kit": "^2.1.1", + "birpc": "^2.5.0", + "debug": "^4.4.1", + "dts-resolver": "^2.1.1", + "get-tsconfig": "^4.10.1" + }, + "engines": { + "node": ">=20.18.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@typescript/native-preview": ">=7.0.0-dev.20250601.1", + "rolldown": "^1.0.0-beta.9", + "typescript": "^5.0.0", + "vue-tsc": "^2.2.0 || ^3.0.0" + }, + "peerDependenciesMeta": { + "@typescript/native-preview": { + "optional": true + }, + "typescript": { + "optional": true + }, + "vue-tsc": { + "optional": true + } + } + }, + "node_modules/safe-content-frame": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/safe-content-frame/-/safe-content-frame-0.0.30.tgz", + "integrity": "sha512-t8XO/59b+YaJCmCpJ3aTZ8TyB6LzG7VzCtUO3u8h0jUsK/wCvQPMro8ioCA6AQr0Ve8Cv9zunFp1VcFGm1aybg==", + "dev": true, + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/semver": { + "version": "7.8.5", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/swr": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.5.1.tgz", + "integrity": "sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tsdown": { + "version": "0.12.9", + "dev": true, + "license": "MIT", + "dependencies": { + "ansis": "^4.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "debug": "^4.4.1", + "diff": "^8.0.2", + "empathic": "^2.0.0", + "hookable": "^5.5.3", + "rolldown": "^1.0.0-beta.19", + "rolldown-plugin-dts": "^0.13.12", + "semver": "^7.7.2", + "tinyexec": "^1.0.1", + "tinyglobby": "^0.2.14", + "unconfig": "^7.3.2" + }, + "bin": { + "tsdown": "dist/run.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@arethetypeswrong/core": "^0.18.1", + "publint": "^0.3.0", + "typescript": "^5.0.0", + "unplugin-lightningcss": "^0.4.0", + "unplugin-unused": "^0.5.0" + }, + "peerDependenciesMeta": { + "@arethetypeswrong/core": { + "optional": true + }, + "publint": { + "optional": true + }, + "typescript": { + "optional": true + }, + "unplugin-lightningcss": { + "optional": true + }, + "unplugin-unused": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unconfig": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "defu": "^6.1.4", + "jiti": "^2.6.1", + "quansync": "^1.0.0", + "unconfig-core": "7.5.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/unconfig-core": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "dev": true, + "license": "MIT" + }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-composed-ref": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/use-composed-ref/-/use-composed-ref-1.4.0.tgz", + "integrity": "sha512-djviaxuOOh7wkj0paeO1Q/4wMZ8Zrnag5H6yBvzN7AKKe8beOaED9SF5/ByLqsku8NP4zQqsvM2u3ew/tJK8/w==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-isomorphic-layout-effect": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.2.1.tgz", + "integrity": "sha512-tpZZ+EX0gaghDAiFR37hj5MgY6ZN55kLiPkJsKxBMZ6GZdOSPJXiOzPM984oPYZ5AnehYx5WQp1+ME8I/P/pRA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-latest": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/use-latest/-/use-latest-1.3.0.tgz", + "integrity": "sha512-mhg3xdm9NaM8q+gLT8KryJPnRFOz1/5XPBhmDEVZK1webPzDjrPk7f/mbpeLqTgB9msytYWANxgALOCJKnLvcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "use-isomorphic-layout-effect": "^1.1.1" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sync-external-store": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zustand": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.15.tgz", + "integrity": "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + } + } +} diff --git a/chat-sdk/package.json b/chat-sdk/package.json new file mode 100644 index 0000000000..a157a5175a --- /dev/null +++ b/chat-sdk/package.json @@ -0,0 +1,108 @@ +{ + "name": "windmill-chat", + "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", + "version": "1.816.0", + "author": "Ruben Fiszel", + "license": "Apache-2.0", + "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/windmill-labs/windmill.git", + "directory": "chat-sdk" + }, + "bugs": { + "url": "https://github.com/windmill-labs/windmill/issues" + }, + "keywords": [ + "windmill", + "chat", + "ai", + "agent", + "react" + ], + "sideEffects": false, + "type": "module", + "main": "dist/index.cjs", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.cjs" + } + }, + "./react": { + "import": { + "types": "./dist/react.d.ts", + "default": "./dist/react.js" + }, + "require": { + "types": "./dist/react.d.ts", + "default": "./dist/react.cjs" + } + }, + "./ai-sdk": { + "import": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.js" + }, + "require": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.cjs" + } + }, + "./assistant-ui": { + "import": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.js" + }, + "require": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.cjs" + } + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsdown && tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "test": "bun test" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + }, + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + } +} diff --git a/chat-sdk/src/ai-sdk.ts b/chat-sdk/src/ai-sdk.ts new file mode 100644 index 0000000000..d93bb99e2f --- /dev/null +++ b/chat-sdk/src/ai-sdk.ts @@ -0,0 +1,349 @@ +import type { ChatTransport, UIMessage, UIMessageChunk, UIMessagePart } from 'ai' +import { + WindmillApiError, + WindmillChatApi, + type FlowConversationMessage, + type WindmillChatApiOptions +} from './api' +import { followJob } from './follow' +import type { AgentStreamEvent } from './stream' +import type { ChatMessage, Conversation } from './types' +import { + conversationIdFor, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + parseJsonOr, + randomId +} from './utils' + +export interface WindmillChatTransportOptions extends WindmillChatApiOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Extra flow inputs sent with every message; `sendMessage(msg, { body })` adds per-message ones. */ + inputs?: Record +} + +export interface WindmillChatTransport + extends ChatTransport { + /** The Windmill conversation id behind an AI SDK chat id (a UUID chat id is used as is). */ + conversationId(chatId: string): string + /** Server history of a chat as `UIMessage`s, oldest first, for `useChat({ messages })`. Needs `flow_conversations:read`. */ + loadMessages(chatId: string, options?: { page?: number; perPage?: number }): Promise + /** The user's conversations for this flow, most recent first. */ + listConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(chatId: string): Promise +} + +interface JobEntry { + jobId: string + offset?: number + done: boolean +} + +/** + * A Vercel AI SDK `ChatTransport` over a chat-mode flow: `useChat({ transport })` + * (and AI Elements, which builds on it) then work against Windmill unchanged. + * The chat id is the conversation, so a UUID id lines up with server history. + */ +export function createWindmillChatTransport( + options: WindmillChatTransportOptions +): WindmillChatTransport { + const api = new WindmillChatApi(options) + // One in-flight or finished job per chat, for `reconnectToStream`. + const jobs = new Map() + + return { + conversationId: conversationIdFor, + + async sendMessages({ chatId, messages, abortSignal, body }) { + const last = messages[messages.length - 1] + if (!last || last.role !== 'user') { + throw new Error('windmill-chat: the last message must be a user message') + } + if (last.parts.some((p) => p.type === 'file')) { + throw new Error( + 'windmill-chat: attachments are not supported; upload the file yourself and pass its reference through `body`' + ) + } + const text = last.parts + .filter((p): p is Extract, { type: 'text' }> => p.type === 'text') + .map((p) => p.text) + .join('\n') + const memoryId = conversationIdFor(chatId) + const jobId = await api.runFlow( + options.flowPath, + { ...options.inputs, ...(body as Record | undefined), user_message: text }, + { memoryId, signal: abortSignal } + ) + const entry: JobEntry = { jobId, done: false } + jobs.set(chatId, entry) + return chunkStream(api, entry, abortSignal) + }, + + async reconnectToStream({ chatId, abortSignal }) { + const entry = jobs.get(chatId) + if (!entry || entry.done) return null + return chunkStream(api, entry, abortSignal) + }, + + async loadMessages(chatId, pagination) { + // A chat that hasn't sent anything yet has no conversation on the server. + const rows = await api.listMessages(conversationIdFor(chatId), pagination).catch((e) => { + if (e instanceof WindmillApiError && e.status === 404) return [] + throw e + }) + return toUIMessages( + rows.map((row) => ({ + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success: row.success ?? true, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + reasoning: row.reasoning ?? undefined, + attachments: row.attachments ?? undefined, + tool: toolFromRow(row) + })) + ) as UI_MESSAGE[] + }, + + async listConversations(pagination) { + const rows = await api.listConversations(options.flowPath, pagination) + return rows.map((row) => ({ + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + })) + }, + + async deleteConversation(chatId) { + await api.deleteConversation(conversationIdFor(chatId)) + jobs.delete(chatId) + } + } +} + +/** The call a stored tool row carries: its tool, named by the sentence the worker words + * every tool row from, and the model's arguments and what it got back — for a failed + * tool, the result is what it failed with. */ +function toolFromRow(row: FlowConversationMessage): ChatMessage['tool'] { + if (row.message_type !== 'tool') return undefined + const name = + /^Used (.+) tool$/.exec(row.content)?.[1] ?? /^Error executing (.+)$/.exec(row.content)?.[1] + if (!name) return undefined + return { + name, + status: (row.success ?? true) ? 'success' : 'error', + arguments: row.tool_arguments ?? undefined, + result: row.tool_result ?? undefined + } +} + +/** Streams a job's answer as AI SDK chunks; resumes from `entry.offset` when the job is already running. */ +function chunkStream( + api: WindmillChatApi, + entry: JobEntry, + signal: AbortSignal | undefined +): ReadableStream { + return new ReadableStream({ + async start(controller) { + const parts = new PartWriter((chunk) => controller.enqueue(chunk)) + parts.emit({ type: 'start' }) + try { + let failure: string | undefined + for await (const event of followJob(api, entry.jobId, { + signal, + streamOffset: entry.offset, + onOffset: (offset) => { + entry.offset = offset + } + })) { + if (event.type === 'stream') { + for (const e of event.events) parts.apply(e) + continue + } + parts.closeOpen() + failure = await failureText(api, entry.jobId, event.result, signal) + if (failure === undefined && !parts.streamedText) { + // No agent streamed: the flow's result is the answer. + const answer = extractChatAnswer(event.result) + if (answer !== undefined) { + parts.text(answer) + parts.closeOpen() + } + } + } + entry.done = true + parts.emit(failure === undefined ? { type: 'finish' } : { type: 'error', errorText: failure }) + } catch (e) { + if (!isAbortError(e)) { + parts.closeOpen() + parts.emit({ type: 'error', errorText: e instanceof Error ? e.message : String(e) }) + } + } finally { + controller.close() + } + } + }) +} + +/** A completed flow's error, when the job did fail (the envelope alone is a legitimate result). */ +async function failureText( + api: WindmillChatApi, + jobId: string, + result: unknown, + signal: AbortSignal | undefined +): Promise { + if (!isErrorResult(result)) return undefined + const failed = await api + .getCompletedResult(jobId, signal) + .then((r) => r.success === false) + .catch(() => true) + return failed ? errorResultMessage(result) : undefined +} + +/** + * Turns agent events into AI SDK chunks. Text and reasoning are open parts that a + * tool call closes (a new round starts new parts); tool calls are `dynamic-tool` + * parts, since the UI declares no tools of its own. + */ +class PartWriter { + streamedText = false + #textId: string | undefined + #reasoningId: string | undefined + #started = new Set() + #inputSent = new Set() + + constructor(readonly emit: (chunk: UIMessageChunk) => void) {} + + text(delta: string): void { + this.streamedText = true + if (!this.#textId) { + this.#textId = randomId() + this.emit({ type: 'text-start', id: this.#textId }) + } + this.emit({ type: 'text-delta', id: this.#textId, delta }) + } + + reasoning(delta: string): void { + if (!this.#reasoningId) { + this.#reasoningId = randomId() + this.emit({ type: 'reasoning-start', id: this.#reasoningId }) + } + this.emit({ type: 'reasoning-delta', id: this.#reasoningId, delta }) + } + + closeOpen(): void { + if (this.#reasoningId) { + this.emit({ type: 'reasoning-end', id: this.#reasoningId }) + this.#reasoningId = undefined + } + if (this.#textId) { + this.emit({ type: 'text-end', id: this.#textId }) + this.#textId = undefined + } + } + + apply(event: AgentStreamEvent): void { + switch (event.type) { + case 'token_delta': + this.text(event.content) + break + case 'reasoning_token_delta': + this.reasoning(event.content) + break + case 'tool_call': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_call_arguments': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: parseJsonOr(event.arguments), + dynamic: true + }) + break + case 'tool_execution': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_result': + this.#toolStart(event.call_id, event.function_name) + if (!this.#inputSent.has(event.call_id)) { + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: undefined, + dynamic: true + }) + } + this.emit( + event.success + ? { type: 'tool-output-available', toolCallId: event.call_id, output: parseJsonOr(event.result), dynamic: true } + : { type: 'tool-output-error', toolCallId: event.call_id, errorText: event.result, dynamic: true } + ) + break + } + } + + #toolStart(callId: string, name: string): void { + if (this.#started.has(callId)) return + this.#started.add(callId) + this.emit({ type: 'tool-input-start', toolCallId: callId, toolName: name, dynamic: true }) + } +} + +/** + * `ChatMessage`s (Windmill's role-per-row model) as `UIMessage`s: an assistant + * turn becomes one message whose parts carry its text, reasoning and tool calls. A + * user message's attachments ride in `metadata.attachments`, as references for + * `WindmillChatApi.attachmentUrl`: a `file` part would need a URL the browser can + * load unauthenticated. + */ +export function toUIMessages(messages: ChatMessage[]): UIMessage[] { + const out: UIMessage[] = [] + for (const m of messages) { + if (m.role === 'user' || m.role === 'system') { + out.push({ + id: m.id, + role: m.role, + ...(m.attachments?.length ? { metadata: { attachments: m.attachments } } : {}), + parts: [{ type: 'text', text: m.content }] + }) + continue + } + let target = out[out.length - 1] + if (!target || target.role !== 'assistant') { + target = { id: m.id, role: 'assistant', parts: [] } + out.push(target) + } + if (m.reasoning) target.parts.push({ type: 'reasoning', text: m.reasoning, state: 'done' }) + if (m.role === 'tool') { + const toolCallId = m.tool?.callId ?? m.id + const toolName = m.tool?.name ?? 'tool' + const input = parseJsonOr(m.tool?.arguments) + target.parts.push( + m.success + ? { type: 'dynamic-tool', toolName, toolCallId, state: 'output-available', input, output: m.tool?.result !== undefined ? parseJsonOr(m.tool.result) : m.content } + : { type: 'dynamic-tool', toolName, toolCallId, state: 'output-error', input, errorText: m.tool?.result ?? m.content } + ) + continue + } + if (m.content) target.parts.push({ type: 'text', text: m.content, state: 'done' }) + } + return out +} diff --git a/chat-sdk/src/api.ts b/chat-sdk/src/api.ts new file mode 100644 index 0000000000..a14e9800a1 --- /dev/null +++ b/chat-sdk/src/api.ts @@ -0,0 +1,364 @@ +import type { ChatAttachment, FetchLike, TokenSource } from './types' + +export interface WindmillChatApiOptions { + baseUrl: string + workspace: string + /** Omit to rely on the session cookie of the Windmill origin. */ + token?: TokenSource + fetch?: FetchLike + /** Server poll interval for a turn's stream (Enterprise; see `ChatOptions.pollDelayMs`). */ + pollDelayMs?: number +} + +export class WindmillApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WindmillApiError' + } +} + +export interface FlowConversation { + id: string + workspace_id: string + flow_path: string + title?: string | null + created_at: string + updated_at: string + created_by: string + /** Started from the flow editor's test panel rather than a deployed run. */ + is_test: boolean +} + +/** + * Which conversations a listing holds: the flow editor's test chats, the deployed flow's + * own (the server's default), or both. + */ +export type ConversationKind = 'test' | 'deployed' | 'all' + +export interface FlowConversationMessage { + id: string + conversation_id: string + message_type: 'user' | 'assistant' | 'system' | 'tool' + content: string + job_id?: string | null + created_at: string + created_seq: number + step_name?: string | null + success?: boolean + /** On a tool row, the arguments the model wrote, without the inputs a step wires in; null for a web search. */ + tool_arguments?: string | null + /** On a tool row, the text the model got back or what the call failed with; a web search's citations. */ + tool_result?: string | null + /** On an answer, the thinking that produced it; on a tool row, the thinking that led to the call. */ + reasoning?: string | null + /** The files a user message carried, as object-storage references. */ + attachments?: ChatAttachment[] | null +} + +export type JobUpdateEvent = + | { + type: 'update' + running?: boolean + completed?: boolean + new_result_stream?: string + stream_offset?: number + only_result?: unknown + flow_stream_job_id?: string + } + | { type: 'error'; error: string } + | { type: 'notfound' } + | { type: 'timeout' } + | { type: 'ping' } + +export interface CompletedJobResult { + completed: boolean + success?: boolean + result?: unknown +} + +/** The part of a flow job's status that names the jobs its steps ran as. */ +export interface FlowJobStatus { + flow_status?: { + modules?: FlowStepStatus[] | null + failure_module?: FlowStepStatus | null + preprocessor_module?: FlowStepStatus | null + } | null +} + +export interface FlowStepStatus { + job?: string | null + flow_jobs?: string[] | null + /** An agent step's rounds; a tool call ran as a job of its own, which its row is persisted under. */ + agent_actions?: { type?: string; job_id?: string | null }[] | null +} + +/** Thin client over the Windmill endpoints a chat-mode flow uses. */ +export class WindmillChatApi { + readonly #baseUrl: string + readonly #workspace: string + readonly #token: TokenSource | undefined + readonly #fetch: FetchLike + readonly #pollDelayMs: number | undefined + + constructor(options: WindmillChatApiOptions) { + this.#baseUrl = normalizeBaseUrl(options.baseUrl) + this.#workspace = options.workspace + this.#token = options.token + this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init)) + this.#pollDelayMs = options.pollDelayMs + } + + /** Starts a turn: runs the flow with `memory_id` set to the conversation id. Returns the job id. */ + async runFlow( + flowPath: string, + args: Record, + options: { memoryId: string; signal?: AbortSignal } + ): Promise { + const res = await this.#request(`jobs/run/f/${encodePath(flowPath)}`, { + method: 'POST', + query: { memory_id: options.memoryId, skip_preprocessor: 'true' }, + body: args, + signal: options.signal + }) + return (await res.text()).trim() + } + + /** + * One server-sent-events connection to a job's updates. The server closes it after + * `TIMEOUT_SSE_STREAM` (a `timeout` event); resume by calling again with the last + * `stream_offset`, never by re-running the flow. + */ + async *streamJob( + jobId: string, + options: { streamOffset?: number; signal?: AbortSignal } = {} + ): AsyncGenerator { + const query: Record = { fast: 'true', only_result: 'true' } + if (this.#pollDelayMs !== undefined) query.poll_delay_ms = String(this.#pollDelayMs) + if (options.streamOffset !== undefined) { + query.stream_offset = String(options.streamOffset) + } + const res = await this.#request(`jobs_u/getupdate_sse/${encodeURIComponent(jobId)}`, { + query, + accept: 'text/event-stream', + signal: options.signal + }) + if (!res.body) { + throw new WindmillApiError('The job update stream has no body', res.status) + } + for await (const data of readServerSentEvents(res.body)) { + try { + yield JSON.parse(data) as JobUpdateEvent + } catch { + // A frame that isn't JSON carries nothing the chat can use. + } + } + } + + async getCompletedResult(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request( + `jobs_u/completed/get_result_maybe/${encodeURIComponent(jobId)}`, + { signal } + ) + return (await res.json()) as CompletedJobResult + } + + /** A flow job with its status: the step job ids are what persisted messages carry as `job_id`. */ + async getFlowJob(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request(`jobs_u/get/${encodeURIComponent(jobId)}`, { + query: { no_logs: 'true' }, + signal + }) + return (await res.json()) as FlowJobStatus + } + + /** + * Where a message's attachment downloads from. The endpoint authenticates like every other + * request: a consumer holding a token must fetch it with that token, not put the URL in an + * `img src`, which would send only the Windmill session cookie. + */ + attachmentUrl(attachment: ChatAttachment): string { + const query = new URLSearchParams({ file_key: attachment.s3 }) + if (attachment.storage) query.set('storage', attachment.storage) + return `${this.#baseUrl}/api/w/${encodeURIComponent(this.#workspace)}/job_helpers/download_s3_file?${query}` + } + + async cancelJob(jobId: string, reason = 'Stopped from the chat'): Promise { + await this.#request(`jobs_u/queue/cancel/${encodeURIComponent(jobId)}`, { + method: 'POST', + body: { reason } + }) + } + + async listConversations( + flowPath: string, + options: { page?: number; perPage?: number; kind?: ConversationKind; signal?: AbortSignal } = {} + ): Promise { + const extra: Record = { flow_path: flowPath } + if (options.kind !== undefined) extra.kind = options.kind + const res = await this.#request('flow_conversations/list', { + query: pagination(options, extra), + signal: options.signal + }) + return (await res.json()) as FlowConversation[] + } + + /** Sets a conversation's title. Its place in the list is kept: only a turn moves one. */ + async renameConversation(conversationId: string, title: string): Promise { + await this.#request(`flow_conversations/update/${encodeURIComponent(conversationId)}`, { + method: 'POST', + body: { title } + }) + } + + /** + * Without `afterSeq`: one page counted from the newest message, returned oldest first. + * With `afterSeq`: the messages created after that cursor, oldest first. + */ + async listMessages( + conversationId: string, + options: { page?: number; perPage?: number; afterSeq?: number; signal?: AbortSignal } = {} + ): Promise { + const extra: Record = {} + if (options.afterSeq !== undefined) extra.after_seq = String(options.afterSeq) + const res = await this.#request( + `flow_conversations/${encodeURIComponent(conversationId)}/messages`, + { query: pagination(options, extra), signal: options.signal } + ) + return (await res.json()) as FlowConversationMessage[] + } + + /** + * Puts bytes in the workspace's object storage under `fileKey` and returns the key they were + * stored under (the server may rewrite it). Needs the workspace to have object storage set. + */ + async uploadFile( + fileKey: string, + body: Blob, + options: { contentType?: string; signal?: AbortSignal } = {} + ): Promise<{ file_key: string }> { + const query: Record = { file_key: fileKey } + if (options.contentType) query.content_type = options.contentType + const res = await this.#request('job_helpers/upload_s3_file', { + method: 'POST', + query, + raw: body, + contentType: options.contentType || 'application/octet-stream', + signal: options.signal + }) + return (await res.json()) as { file_key: string } + } + + async deleteConversation(conversationId: string): Promise { + await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, { + method: 'DELETE' + }) + } + + async #request( + path: string, + init: { + method?: string + query?: Record + /** JSON-encoded. */ + body?: unknown + /** Sent as is, under `contentType`. */ + raw?: Blob + contentType?: string + accept?: string + signal?: AbortSignal + } = {} + ): Promise { + const url = new URL(`${this.#baseUrl}/api/w/${encodeURIComponent(this.#workspace)}/${path}`) + for (const [k, v] of Object.entries(init.query ?? {})) url.searchParams.set(k, v) + + const headers: Record = {} + if (init.accept) headers['Accept'] = init.accept + if (init.body !== undefined) headers['Content-Type'] = 'application/json' + else if (init.raw !== undefined) headers['Content-Type'] = init.contentType ?? 'application/octet-stream' + const token = typeof this.#token === 'function' ? await this.#token() : this.#token + if (token) headers['Authorization'] = `Bearer ${token}` + + const res = await this.#fetch(url.toString(), { + method: init.method ?? 'GET', + headers, + body: init.body === undefined ? init.raw : JSON.stringify(init.body), + // A token must not be paired with ambient cookies; without one, the cookie is + // the credential and only rides same-origin requests. + credentials: token ? 'omit' : 'same-origin', + signal: init.signal + }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new WindmillApiError( + `${init.method ?? 'GET'} ${path} failed (${res.status})${text ? `: ${text}` : ''}`, + res.status + ) + } + return res + } +} + +export function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '') +} + +function encodePath(path: string): string { + return path.split('/').map(encodeURIComponent).join('/') +} + +function pagination( + options: { page?: number; perPage?: number }, + extra: Record +): Record { + const query = { ...extra } + if (options.page !== undefined) query.page = String(options.page) + if (options.perPage !== undefined) query.per_page = String(options.perPage) + return query +} + +/** Yields the `data` payload of each event in a `text/event-stream` body. */ +export async function* readServerSentEvents( + body: ReadableStream +): AsyncGenerator { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = '' + // A CR ending a chunk may be half of a CRLF; it waits for the next chunk. + let carry = '' + try { + while (true) { + const { value, done } = await reader.read() + if (done) break + let text = carry + decoder.decode(value, { stream: true }) + carry = '' + if (text.endsWith('\r')) { + carry = '\r' + text = text.slice(0, -1) + } + buffer += text.replace(/\r\n?/g, '\n') + let end: number + while ((end = buffer.indexOf('\n\n')) !== -1) { + const data = eventData(buffer.slice(0, end)) + buffer = buffer.slice(end + 2) + if (data !== undefined) yield data + } + } + if (carry) buffer += '\n' + const data = eventData(buffer) + if (data !== undefined) yield data + } finally { + // Closes the connection when the consumer stops early. + reader.cancel().catch(() => {}) + } +} + +function eventData(block: string): string | undefined { + const lines = block + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(line.startsWith('data: ') ? 6 : 5)) + return lines.length > 0 ? lines.join('\n') : undefined +} diff --git a/chat-sdk/src/assistant-ui.ts b/chat-sdk/src/assistant-ui.ts new file mode 100644 index 0000000000..b9a76db679 --- /dev/null +++ b/chat-sdk/src/assistant-ui.ts @@ -0,0 +1,126 @@ +import { + useExternalStoreRuntime, + type AppendMessage, + type AssistantRuntime, + type ExternalStoreAdapter, + type ThreadMessageLike +} from '@assistant-ui/react' +import { useEffect, useMemo } from 'react' +import { useWindmillChat } from './react' +import type { ChatMessage, ChatOptions } from './types' +import { parseJsonOr } from './utils' + +/** One assistant turn: the assistant and tool rows between two user messages. */ +export interface WindmillTurn { + id: string + role: 'user' | 'assistant' | 'system' + messages: ChatMessage[] +} + +export interface WindmillRuntimeOptions extends ChatOptions { + /** Load the conversation list on mount so `ThreadListPrimitive` has something to show. Default true. */ + threadList?: boolean +} + +/** + * An assistant-ui runtime over a chat-mode flow, for `AssistantRuntimeProvider`. + * Conversations become threads, so the thread list primitives switch, create and + * delete Windmill conversations. + */ +export function useWindmillRuntime(options: WindmillRuntimeOptions): AssistantRuntime { + const chat = useWindmillChat(options) + const turns = useMemo(() => groupTurns(chat.messages), [chat.messages]) + const withThreadList = options.threadList !== false + useEffect(() => { + if (withThreadList) chat.loadConversations().catch(() => {}) + }, [chat.chat, withThreadList]) + + const adapter: ExternalStoreAdapter = { + messages: turns, + isRunning: chat.status === 'submitted' || chat.status === 'streaming', + convertMessage: toThreadMessage, + onNew: async (message: AppendMessage) => { + if (message.role !== 'user') return + await chat.sendMessage(appendedText(message)) + }, + onCancel: async () => { + await chat.stop() + }, + adapters: withThreadList + ? { + threadList: { + threadId: chat.conversationId, + threads: chat.conversations.map((c) => ({ status: 'regular' as const, id: c.id, title: c.title })), + onSwitchToNewThread: () => chat.newConversation(), + onSwitchToThread: (id) => chat.selectConversation(id), + onDelete: (id) => chat.deleteConversation(id), + onRename: (id, title) => chat.renameConversation(id, title) + } + } + : undefined + } + return useExternalStoreRuntime(adapter) +} + +function appendedText(message: AppendMessage): string { + return message.content + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join('\n') +} + +/** Folds the role-per-row message list into turns: one entry per user message, one per answer. */ +export function groupTurns(messages: ChatMessage[]): WindmillTurn[] { + const turns: WindmillTurn[] = [] + for (const m of messages) { + const last = turns[turns.length - 1] + if (m.role === 'user' || m.role === 'system' || !last || last.role !== 'assistant') { + turns.push({ id: m.id, role: m.role === 'tool' ? 'assistant' : m.role, messages: [m] }) + } else { + last.messages.push(m) + } + } + return turns +} + +/** A turn as assistant-ui content parts; the status reflects streaming and a failed flow. */ +export function toThreadMessage(turn: WindmillTurn): ThreadMessageLike { + const first = turn.messages[0] + const createdAt = new Date(first.createdAt) + if (turn.role !== 'assistant') { + return { id: turn.id, role: turn.role, createdAt, content: [{ type: 'text', text: first.content }] } + } + const content: ThreadContentPart[] = [] + for (const m of turn.messages) { + if (m.reasoning) content.push({ type: 'reasoning', text: m.reasoning }) + if (m.role === 'tool') { + const args = parseJsonOr(m.tool?.arguments) + content.push({ + type: 'tool-call', + toolCallId: m.tool?.callId ?? m.id, + toolName: m.tool?.name ?? 'tool', + args: (isJsonObject(args) ? args : args === undefined ? {} : { input: args }) as ToolCallArgs, + argsText: m.tool?.arguments ?? '', + result: + m.tool?.status === 'running' ? undefined : m.tool?.result !== undefined ? parseJsonOr(m.tool.result) : m.content, + isError: m.tool?.status === 'error' + }) + continue + } + if (m.content) content.push({ type: 'text', text: m.content }) + } + const last = turn.messages[turn.messages.length - 1] + const status: ThreadMessageLike['status'] = turn.messages.some((m) => m.pending) + ? { type: 'running' } + : last.role === 'assistant' && !last.success + ? { type: 'incomplete', reason: 'error', error: last.content } + : { type: 'complete', reason: 'stop' } + return { id: turn.id, role: 'assistant', createdAt, content, status } +} + +type ThreadContentPart = Exclude[number] +type ToolCallArgs = Extract['args'] + +function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/chat-sdk/src/attachments.ts b/chat-sdk/src/attachments.ts new file mode 100644 index 0000000000..ac5e71c7d3 --- /dev/null +++ b/chat-sdk/src/attachments.ts @@ -0,0 +1,116 @@ +import type { WindmillChatApi } from './api' +import type { AttachmentUpload } from './types' +import { abortError, isAbortError } from './utils' + +/** + * Where a chat's uploads live in the workspace's object storage. Under `windmill_uploads/` + * because the default Enterprise storage permissions grant every user write and read there + * and deny any other top-level prefix: a key outside it is refused for non-admins, both on + * upload and when the agent's job reads the file back. + */ +export const CHAT_UPLOADS_PREFIX = 'windmill_uploads/chat' + +/** What an AI agent step reads out of `user_attachments`. */ +export interface UploadedAttachment { + s3: string + filename: string +} + +/** + * The extension each type must be stored under. The worker reads an attachment's media type + * from the key's extension only (`mime_guess` in `windmill-ai/src/image_handler.rs`, falling + * back to `image/png`), never from the stored content type, so the extension must be true. + */ +const EXTENSION_BY_MEDIA_TYPE: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'application/pdf': 'pdf' +} + +/** + * The name an attachment is stored under: the picked name with the extension its media type + * needs, e.g. a `photo.webp` re-encoded to PNG becomes `photo.png`. Other types keep their name. + */ +export function storedAttachmentName(filename: string, mediaType: string): string { + const extension = EXTENSION_BY_MEDIA_TYPE[mediaType] + if (!extension) return filename + const stem = filename.replace(/\.[^./]+$/, '') + return `${stem || filename}.${extension}` +} + +/** The bytes of an attachment as a Blob carrying its media type. */ +export function attachmentBlob(attachment: AttachmentUpload): Blob { + const data = + typeof attachment.data === 'string' + ? dataUrlToBlob(attachment.data, 'application/octet-stream') + : attachment.data + return attachment.mediaType && attachment.mediaType !== data.type + ? new Blob([data], { type: attachment.mediaType }) + : data +} + +function dataUrlToBlob(dataUrl: string, fallbackType: string): Blob { + const comma = dataUrl.indexOf(',') + if (!dataUrl.startsWith('data:') || comma === -1) { + throw new Error('windmill-chat: an attachment given as a string must be a data: URL') + } + const header = dataUrl.slice(5, comma) + const isBase64 = header.endsWith(';base64') + const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType + const payload = dataUrl.slice(comma + 1) + if (!isBase64) return new Blob([decodeURIComponent(payload)], { type: mediaType }) + const binary = atob(payload) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new Blob([bytes], { type: mediaType }) +} + +/** + * Put each attachment in the workspace's object storage and hand back what the agent reads. + * The key's turn prefix and per-file index keep two files with the same name, in this turn or + * an earlier one, from overwriting each other; the name stays the last segment. + */ +export async function uploadAttachments( + api: WindmillChatApi, + attachments: AttachmentUpload[], + turnId: string, + signal?: AbortSignal +): Promise { + const prefix = `${CHAT_UPLOADS_PREFIX}/${turnId}` + // One failed upload aborts the rest. Nothing already stored is deleted: the chat never + // removes objects from the workspace's storage, so a send that does not run leaves them. + if (signal?.aborted) throw abortError() + const batch = new AbortController() + const abortBatch = () => batch.abort() + signal?.addEventListener('abort', abortBatch, { once: true }) + try { + const results = await Promise.allSettled( + attachments.map(async (attachment, index) => { + try { + const blob = attachmentBlob(attachment) + const filename = storedAttachmentName( + attachment.name || `attachment-${index + 1}`, + blob.type + ) + const { file_key } = await api.uploadFile(`${prefix}/${index}/${filename}`, blob, { + contentType: blob.type, + signal: batch.signal + }) + return { s3: file_key, filename } + } catch (e) { + batch.abort() + throw e + } + }) + ) + const reasons = results.flatMap((r) => (r.status === 'rejected' ? [r.reason] : [])) + // A stop that lands once every upload has answered still withdraws the batch. + if (reasons.length === 0 && !signal?.aborted) { + return results.flatMap((r) => (r.status === 'fulfilled' ? [r.value] : [])) + } + // The failure that started it, not the aborts it caused in the other uploads. + throw reasons.find((reason) => !isAbortError(reason)) ?? reasons[0] ?? abortError() + } finally { + signal?.removeEventListener('abort', abortBatch) + } +} diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts new file mode 100644 index 0000000000..2e2c52f3df --- /dev/null +++ b/chat-sdk/src/chat.ts @@ -0,0 +1,925 @@ +import { + WindmillApiError, + WindmillChatApi, + type ConversationKind, + type FlowConversation, + type FlowConversationMessage +} from './api' +import { resolveConfig, type ResolvedConfig } from './config' +import { followJob } from './follow' +import { createLocalHistory, type LocalHistory } from './history' +import { uploadAttachments } from './attachments' +import type { AgentStreamEvent } from './stream' +import type { + Chat, + ChatMessage, + ChatOptions, + ChatState, + Conversation, + SendMessageOptions, + ToolInvocation +} from './types' +import { + conversationTitle, + truncateTitle, + errorResultMessage, + extractChatAnswer, + abortError, + isAbortError, + isErrorResult, + now, + randomId, + sleep +} from './utils' + +const POLL_INTERVAL_MS = 1000 +/** Local history mirrors the state; a stream of deltas is coalesced into one write. */ +const PERSIST_DEBOUNCE_MS = 250 +/** Messages persist from spawned tasks that can land just after the flow completes. */ +const RECONCILE_ATTEMPTS = 3 +const RECONCILE_DELAY_MS = 400 + +interface Turn { + controller: AbortController + conversationId: string + /** Id of the turn's user message; the answer is whatever follows it. */ + userMessageId: string + /** The turn opened the conversation; withdrawing it closes the conversation again. */ + isNew: boolean + /** The run was asked for. Before that, a failure or a stop withdraws the turn instead of failing it. */ + started: boolean + /** Both `stop()` and the send's own rejection withdraw; only the first may. */ + withdrawn: boolean + jobId?: string + /** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */ + jobIds?: Set + /** Id of the streaming assistant message; cleared when a tool call ends the round. */ + assistantId?: string + streamedText: boolean +} + +export function createChat(options: ChatOptions): Chat { + return new ChatImpl(options) +} + +class ChatImpl implements Chat { + readonly #config: ResolvedConfig + readonly #api: WindmillChatApi + readonly #local: LocalHistory + readonly #listeners = new Set<(state: ChatState) => void>() + #state: ChatState + #turn: Turn | undefined + #page = 1 + /** The kind the caller last listed, so the refresh after a new turn lists the same rows. */ + #conversationKind: ConversationKind | undefined + #persistTimer: ReturnType | undefined + + constructor(options: ChatOptions) { + this.#config = resolveConfig(options) + this.#api = new WindmillChatApi({ + baseUrl: this.#config.baseUrl, + workspace: this.#config.workspace, + token: this.#config.token, + fetch: this.#config.fetch, + pollDelayMs: this.#config.pollDelayMs + }) + this.#local = createLocalHistory( + this.#config.storage, + `windmill-chat:${this.#config.baseUrl}:${this.#config.workspace}:${this.#config.flowPath}` + + (this.#config.storageKey ? `:${this.#config.storageKey}` : '') + ) + this.#state = { + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + conversations: [], + history: this.#config.history, + loadingMessages: false, + hasMoreMessages: false + } + } + + getState = (): ChatState => this.#state + + subscribe = (listener: (state: ChatState) => void): (() => void) => { + this.#listeners.add(listener) + listener(this.#state) + return () => { + this.#listeners.delete(listener) + } + } + + sendMessage = async (text: string, options: SendMessageOptions = {}): Promise => { + const content = text.trim() + if (!content) { + // A run needs a message; files alone would otherwise be dropped without a word. + if (options.attachments?.length) throw new Error('windmill-chat: attachments need a message to go with them') + return + } + if (this.#turn) { + throw new Error('windmill-chat: a message is already being answered; call stop() first') + } + const attachments = options.attachments ?? [] + const attachmentsInput = options.attachmentsInput + if (attachments.length > 0 && !attachmentsInput) { + throw new Error('windmill-chat: attachments need `attachmentsInput`, the flow input that takes them') + } + if (attachmentsInput && !attachmentsInput.multiple && attachments.length > 1) { + // Uploading all of them would run with the first and leave the rest stranded in storage. + throw new Error(`windmill-chat: \`${attachmentsInput.name}\` holds one file; got ${attachments.length}`) + } + const isNew = this.#state.conversationId === undefined + const conversationId = this.#state.conversationId ?? randomId() + const turn: Turn = { + controller: new AbortController(), + conversationId, + userMessageId: `pending-${randomId()}`, + isNew, + started: false, + withdrawn: false, + streamedText: false + } + this.#turn = turn + + const timestamp = now() + const conversation: Conversation = this.#state.conversations.find( + (c) => c.id === conversationId + ) ?? { id: conversationId, title: conversationTitle(content), createdAt: timestamp, updatedAt: timestamp } + const touched = { ...conversation, updatedAt: timestamp } + this.#set({ + conversationId, + messages: [ + ...this.#state.messages, + { id: turn.userMessageId, role: 'user', content, success: true, createdAt: timestamp, pending: true } + ], + status: 'submitted', + error: undefined + }) + + try { + const args: Record = { ...this.#config.inputs, ...options.inputs, user_message: content } + if (attachmentsInput && attachments.length > 0) { + // Uploaded with the turn already shown as submitted: the message is in the transcript + // and `stop()` can abort the upload, while a second send is refused as usual. + const uploaded = await uploadAttachments(this.#api, attachments, randomId(), turn.controller.signal) + args[attachmentsInput.name] = attachmentsInput.multiple ? uploaded : uploaded[0] + // Shown on the pending message until its server row replaces it, carrying its own. + const carried = uploaded.map((u) => ({ input: attachmentsInput.name, s3: u.s3, filename: u.filename })) + if (this.#turnActive(turn)) { + this.#set({ + messages: this.#state.messages.map((m) => (m.id === turn.userMessageId ? { ...m, attachments: carried } : m)) + }) + } + } + // Nothing may start once stop() or a conversation switch has withdrawn the turn, including + // a stop from a subscriber told of the attachments just above. + if (turn.controller.signal.aborted) throw abortError() + turn.started = true + // Listed only once the run is asked for: a send that never runs (an upload that failed + // or was stopped) then has no conversation entry to take back. + this.#set({ + conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)] + }) + this.#rememberConversation() + const context = { memoryId: conversationId, conversationId, signal: turn.controller.signal } + turn.jobId = this.#config.run + ? await this.#config.run(args, context) + : await this.#api.runFlow(this.#config.flowPath, args, context) + const stopPolling = this.#state.history === 'server' ? this.#startPolling(turn) : () => {} + let result: unknown + try { + result = await this.#follow(turn, stopPolling) + } finally { + stopPolling() + } + await this.#finishTurn(turn, result, isNew) + } catch (e) { + if (!turn.started) { + // Nothing ran: the message is withdrawn rather than shown as a failed turn, and the + // caller gets the reason (an upload that failed, or the AbortError of a stop()). + if (this.#turn === turn) this.#turn = undefined + this.#withdrawTurn(turn) + throw e + } + // stop() and a conversation switch abort the turn and settle the state themselves. + if (turn.controller.signal.aborted || isAbortError(e)) return + this.#failTurn(turn, e) + } finally { + if (this.#turn === turn) this.#turn = undefined + } + } + + stop = async (): Promise => { + const turn = this.#turn + if (!turn) return + this.#detachTurn() + if (!turn.started) { + // Still uploading its attachments: there is no run to cancel, and the message the + // reader took back must not stay in the transcript as sent. + this.#withdrawTurn(turn) + return + } + if (this.#state.conversationId === turn.conversationId) { + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + this.#persistLocal() + } + if (turn.jobId) { + // Needs `jobs:write` on a token; the stream is closed either way. + await this.#api.cancelJob(turn.jobId).catch(() => {}) + } + if (this.#state.history !== 'server') return + // A cancelled flow persists its failure as the assistant's answer. Picked up only + // while the conversation is still idle: a turn started meanwhile owns the state. + await sleep(RECONCILE_DELAY_MS).catch(() => {}) + if (this.#turn || this.#state.conversationId !== turn.conversationId) return + await this.#syncFromServer(turn.conversationId).catch(() => {}) + } + + newConversation = (): void => { + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: false, + hasMoreMessages: false + }) + } + + selectConversation = async (conversationId: string): Promise => { + if (conversationId === this.#state.conversationId) return + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: true, + hasMoreMessages: false + }) + if (this.#state.history !== 'server') { + this.#set({ + messages: this.#state.history === 'local' ? this.#local.getMessages(conversationId) : [], + loadingMessages: false + }) + return + } + try { + const rows = await this.#api.listMessages(conversationId, { + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + this.#set({ + messages: rows.map(fromRow), + loadingMessages: false, + hasMoreMessages: rows.length === this.#config.pageSize + }) + } catch (e) { + if (this.#state.conversationId !== conversationId) return + if (this.#fallBackToLocal(e)) { + this.#set({ messages: this.#local.getMessages(conversationId), loadingMessages: false }) + return + } + this.#set({ loadingMessages: false, status: 'error', error: toError(e) }) + } + } + + loadConversations = async ( + options: { page?: number; perPage?: number; kind?: ConversationKind } = {} + ): Promise => { + const page = options.page ?? 1 + // A different kind is a different listing: its first rows replace the held ones, on + // whichever page they were asked for. + const kindChanged = 'kind' in options && options.kind !== this.#conversationKind + if ('kind' in options) this.#conversationKind = options.kind + const kind = this.#conversationKind + let conversations: Conversation[] + if (this.#state.history === 'server') { + try { + const rows = await this.#api.listConversations(this.#config.flowPath, { + page, + perPage: options.perPage ?? this.#config.pageSize, + kind + }) + conversations = rows.map(fromConversation) + } catch (e) { + if (!this.#fallBackToLocal(e)) throw e + conversations = this.#local.listConversations() + } + } else { + conversations = this.#state.history === 'local' ? this.#local.listConversations() : [] + } + // Another kind was asked for while this list was on its way: its rows are not the + // listing any more, whichever response lands last. + if (kind !== this.#conversationKind) return conversations + const known = new Set(this.#state.conversations.map((c) => c.id)) + this.#set({ + conversations: + page === 1 || kindChanged + ? conversations + : [...this.#state.conversations, ...conversations.filter((c) => !known.has(c.id))] + }) + return conversations + } + + deleteConversation = async (conversationId: string): Promise => { + if (this.#state.conversationId === conversationId) { + // Nothing of the current turn may be written back under the deleted id. + this.#detachTurn() + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + this.newConversation() + } + if (this.#state.history === 'server') { + await this.#api.deleteConversation(conversationId) + } else if (this.#state.history === 'local') { + this.#local.deleteConversation(conversationId) + } + this.#set({ conversations: this.#state.conversations.filter((c) => c.id !== conversationId) }) + } + + renameConversation = async (conversationId: string, title: string): Promise => { + // Cut here as the server cuts, so the title shown is the one stored. + const trimmed = truncateTitle(title.trim()) + if (!trimmed) return + if (this.#state.history === 'server') { + await this.#api.renameConversation(conversationId, trimmed) + } else if (this.#state.history === 'local') { + this.#local.renameConversation(conversationId, trimmed) + } + // Patched in place: the server keeps `updated_at` on a rename, so the list order the + // next load returns is the one shown now. + this.#set({ + conversations: this.#state.conversations.map((c) => + c.id === conversationId ? { ...c, title: trimmed } : c + ) + }) + } + + loadOlderMessages = async (): Promise => { + const conversationId = this.#state.conversationId + if ( + !conversationId || + this.#state.history !== 'server' || + !this.#state.hasMoreMessages || + this.#state.loadingMessages + ) { + return + } + const page = this.#page + 1 + this.#set({ loadingMessages: true }) + try { + const rows = await this.#api.listMessages(conversationId, { + page, + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + const known = new Set(this.#state.messages.map((m) => m.serverId ?? m.id)) + this.#page = page + this.#set({ + messages: [...rows.map(fromRow).filter((m) => !known.has(m.id)), ...this.#state.messages], + hasMoreMessages: rows.length === this.#config.pageSize + }) + } finally { + if (this.#state.conversationId === conversationId) this.#set({ loadingMessages: false }) + } + } + + destroy = (): void => { + this.#leaveConversation() + } + + // ---- turn internals ---- + + async #follow(turn: Turn, onStreamStart: () => void): Promise { + let started = false + for await (const event of followJob(this.#api, turn.jobId!, { signal: turn.controller.signal })) { + if (event.type === 'completed') return event.result + if (!started) { + started = true + // Persisted rows for the streaming step would duplicate what is streaming. + onStreamStart() + } + this.#applyEvents(turn, event.events) + } + throw new Error('windmill-chat: the job stream ended before the flow completed') + } + + #applyEvents(turn: Turn, events: AgentStreamEvent[]): void { + if (events.length === 0 || !this.#turnActive(turn)) return + let messages = [...this.#state.messages] + const upsertTool = ( + callId: string, + name: string, + patch: Partial & { content?: string; success?: boolean } + ) => { + const { content, success, ...toolPatch } = patch + // Only this turn's tool messages are pending; a provider may reuse call ids across turns. + const i = messages.findIndex( + (m) => m.pending && m.role === 'tool' && m.tool?.callId === callId + ) + if (i >= 0) { + const existing = messages[i] + messages[i] = { + ...existing, + content: content ?? existing.content, + success: success ?? existing.success, + tool: { ...existing.tool!, ...toolPatch } + } + } else { + // Thinking that produced no text led to this call, and is stored on its row. + const a = turn.assistantId ? messages.findIndex((m) => m.id === turn.assistantId) : -1 + const reasoning = a >= 0 && messages[a].content === '' ? messages.splice(a, 1)[0].reasoning : undefined + messages.push({ + id: `pending-${randomId()}`, + role: 'tool', + content: content ?? '', + reasoning, + success: success ?? true, + createdAt: now(), + pending: true, + tool: { callId, name, status: 'running', ...toolPatch } + }) + } + turn.assistantId = undefined + } + const appendAssistant = (text: string, reasoning: string) => { + const i = turn.assistantId + ? messages.findIndex((m) => m.id === turn.assistantId) + : -1 + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...m, + content: m.content + text, + reasoning: reasoning ? (m.reasoning ?? '') + reasoning : m.reasoning + } + } else { + turn.assistantId = `pending-${randomId()}` + messages.push({ + id: turn.assistantId, + role: 'assistant', + content: text, + reasoning: reasoning || undefined, + success: true, + createdAt: now(), + pending: true + }) + } + } + for (const event of events) { + switch (event.type) { + case 'token_delta': + turn.streamedText = true + appendAssistant(event.content, '') + break + case 'reasoning_token_delta': + appendAssistant('', event.content) + break + // A call completes the round's text: text after it is a new message. + case 'tool_call': + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_call_arguments': + upsertTool(event.call_id, event.function_name, { arguments: event.arguments }) + break + case 'tool_execution': + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_result': + upsertTool(event.call_id, event.function_name, { + status: event.success ? 'success' : 'error', + result: event.result, + success: event.success, + // The same text Windmill persists for the tool message. + content: event.success + ? `Used ${event.function_name} tool` + : `Error executing ${event.function_name}` + }) + break + } + } + this.#set({ messages, status: 'streaming' }) + } + + async #finishTurn(turn: Turn, result: unknown, isNew: boolean): Promise { + if (!this.#turnActive(turn)) return + if (this.#state.history === 'server') { + turn.jobIds = await this.#turnJobIds(turn) + if (!this.#turnActive(turn)) return + const reconciled = await this.#reconcileTurn(turn) + if (!this.#turnActive(turn)) return + if (reconciled) { + this.#set({ status: 'idle' }) + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + if (isNew) await this.loadConversations().catch(() => {}) + return + } + // Server history just proved unreadable: the turn completes as local history. + } + let messages = this.#state.messages + let failed = false + if (isErrorResult(result)) { + // The envelope is also a legitimate result shape; the job's own status decides. + failed = await this.#api + .getCompletedResult(turn.jobId!, turn.controller.signal) + .then((r) => r.success === false) + .catch(() => true) + if (!this.#turnActive(turn)) return + if (failed) { + messages = [...messages, assistantMessage(errorResultMessage(result), false, turn.jobId)] + } + } + if (!failed && !turn.streamedText) { + const answer = extractChatAnswer(result) + if (answer !== undefined) { + messages = [...messages, assistantMessage(answer, true, turn.jobId)] + } + } + this.#set({ messages: finalized(messages), status: 'idle' }) + this.#persistLocal() + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + } + + /** + * Folds what the server persisted for the turn into the message list. The rows + * are written by the worker in their own transactions, each of which can trail + * the flow's completion, so a streamed message whose row hasn't landed stays and + * the list is re-read a few times before the rest is kept as streamed. + * Returns false when the server holds no answer for the turn: history fell back to + * local, the read was refused or kept failing, or no assistant row has landed. The + * caller then finishes the turn from the flow result, so an answer is never lost + * to history. Whether a row counts is read from the message list, not from what + * this read returned: the turn's polling may have merged the answer already. + */ + async #reconcileTurn(turn: Turn): Promise { + const answered = () => this.#answered(turn) + for (let attempt = 1; attempt <= RECONCILE_ATTEMPTS; attempt++) { + let rows: FlowConversationMessage[] + try { + rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal: turn.controller.signal + }) + } catch (e) { + if (isAbortError(e)) throw e + if (this.#fallBackToLocal(e)) return false + const refused = e instanceof WindmillApiError && (e.status === 401 || e.status === 403) + if (refused || attempt === RECONCILE_ATTEMPTS) return answered() + await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + continue + } + if (!this.#turnActive(turn)) return true + this.#mergeRows(rows) + if (answered() && !this.#state.messages.some((m) => m.pending && m.content)) break + if (attempt < RECONCILE_ATTEMPTS) await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + } + if (!this.#turnActive(turn)) return true + this.#set({ messages: finalized(this.#state.messages) }) + return answered() + } + + /** + * The latest row the turn persisted after its user message is an assistant + * message. An agent issues each round's text row before that round's tool rows, + * and a tool row when the tool finishes, so an earlier round's text is followed + * by a tool row and only the answer closes the turn (the inserts are spawned, so + * a badly delayed one can invert that order at the cost of the reconcile + * retries). The content is not compared with the flow result: an image answer, a + * structured one and a forwarded agent result are all persisted in a shape the + * result does not reproduce. Rows carrying a job id belong to the turn when the + * job is one of the turn's, which leaves out an earlier turn whose job outlived + * `stop()` (a token without `jobs:write` cannot cancel it); a tool row without one + * (an MCP call runs inside the agent step) belongs to whatever turn is under way. + */ + #answered(turn: Turn): boolean { + const messages = this.#state.messages + const from = messages.findIndex((m) => m.id === turn.userMessageId) + const ownJob = (m: ChatMessage) => + turn.jobIds === undefined || (m.jobId === undefined ? m.role === 'tool' : turn.jobIds.has(m.jobId)) + let latest: ChatMessage | undefined + for (let i = from + 1; i < messages.length; i++) { + const m = messages[i] + if (m.seq === undefined || m.role === 'user' || !ownJob(m)) continue + if (latest === undefined || m.seq > latest.seq!) latest = m + } + return latest?.role === 'assistant' + } + + /** + * The flow job plus every step job it ran, the failure and preprocessor steps + * included (a failure handler's answer is persisted under its own job), and the + * jobs an agent step's tool calls ran as (a tool row is persisted under its own + * job too). Unknown when the read fails. + */ + async #turnJobIds(turn: Turn): Promise | undefined> { + try { + const job = await this.#api.getFlowJob(turn.jobId!, turn.controller.signal) + const ids = new Set([turn.jobId!]) + const status = job.flow_status + for (const m of [...(status?.modules ?? []), status?.failure_module, status?.preprocessor_module]) { + if (m?.job) ids.add(m.job) + for (const j of m?.flow_jobs ?? []) ids.add(j) + for (const a of m?.agent_actions ?? []) if (a.job_id) ids.add(a.job_id) + } + return ids + } catch (e) { + if (isAbortError(e)) throw e + return undefined + } + } + + /** + * Take back the user message of a turn that never ran. A conversation it would have opened + * was never listed (see `sendMessage`), so only the message goes, and, while it is the turn + * on screen, the busy status. A switch away mid-upload has already written the message to + * local history, so it is removed there too. + */ + #withdrawTurn(turn: Turn): void { + if (turn.withdrawn) return + turn.withdrawn = true + const id = turn.conversationId + const withoutTurn = (messages: ChatMessage[]) => messages.filter((m) => m.id !== turn.userMessageId) + if (this.#state.conversationId === id) { + const messages = withoutTurn(this.#state.messages) + // A turn started since, such as a resend right after Stop, owns the status. + const newerTurn = this.#turn !== undefined && this.#turn !== turn + if (newerTurn) { + this.#set({ messages }) + } else { + const unopened = turn.isNew && messages.length === 0 + this.#set({ messages, status: 'idle', error: undefined, ...(unopened ? { conversationId: undefined } : {}) }) + } + this.#persistLocal() + } + if (this.#state.history === 'local' && this.#state.conversationId !== id) { + const stored = withoutTurn(this.#local.getMessages(id)) + if (stored.length > 0) this.#local.saveMessages(id, stored) + else if (!this.#state.conversations.some((c) => c.id === id)) this.#local.deleteConversation(id) + } + } + + #failTurn(turn: Turn, e: unknown): void { + if (!this.#turnActive(turn)) return + const error = toError(e) + this.#set({ + messages: [...finalized(this.#state.messages), assistantMessage(error.message, false, turn.jobId)], + status: 'error', + error + }) + this.#persistLocal() + this.#config.onError?.(error, { conversationId: turn.conversationId, jobId: turn.jobId }) + } + + /** Before the answer streams, earlier steps may already have persisted messages. */ + #startPolling(turn: Turn): () => void { + let stopped = false + const { signal } = turn.controller + const loop = async () => { + while (!stopped) { + try { + await sleep(POLL_INTERVAL_MS, signal) + } catch { + return + } + if (stopped) return + try { + const rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal + }) + if (!stopped && this.#turnActive(turn)) this.#mergeRows(rows) + } catch { + // transient; the completion reconciliation catches up + } + } + } + void loop() + return () => { + stopped = true + } + } + + async #syncFromServer(conversationId: string): Promise { + const rows = await this.#api.listMessages(conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100 + }) + if (this.#turn || this.#state.conversationId !== conversationId) return + this.#mergeRows(rows) + this.#set({ messages: finalized(this.#state.messages) }) + } + + /** + * Folds persisted rows into the message list. A row standing for a message the + * client already shows (same role and text; for a tool, the same tool name, since + * the server words a failure differently) takes its place under the client's id + * and keeps what only the stream knew: reasoning, call id, arguments, result. + * Other rows append in server order. Nothing is dropped: a streamed message + * outlives a row that never lands. + */ + #mergeRows(rows: FlowConversationMessage[]): void { + if (rows.length === 0) return + const messages = [...this.#state.messages] + const known = new Set(messages.map((m) => m.serverId ?? m.id)) + for (const row of rows.map(fromRow)) { + if (known.has(row.id)) continue + known.add(row.id) + let i = messages.findIndex( + (m) => + m.seq === undefined && + m.role === row.role && + (m.content === row.content || (row.tool !== undefined && m.tool?.name === row.tool.name)) + ) + // A structured answer streams as the call of the structured-output tool, whose + // arguments are the answer's text: its row replaces that call. Only past the newest + // user message, where a stopped turn's identical call cannot be. + if (i < 0 && row.role === 'assistant') { + let j = messages.length - 1 + while (j >= 0 && messages[j].role !== 'user') { + const m = messages[j] + if (m.seq === undefined && m.role === 'tool' && m.tool?.arguments === row.content) i = j + j-- + } + } + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...row, + id: m.id, + reasoning: m.reasoning ?? row.reasoning, + // The stream's call wins where it has a value; a stream cut short leaves gaps the row fills. + tool: + row.role === 'tool' && m.tool + ? { + ...m.tool, + arguments: m.tool.arguments ?? row.tool?.arguments, + result: m.tool.result ?? row.tool?.result, + status: row.tool?.status ?? m.tool.status + } + : row.tool + } + } else { + // A tool row nothing streamed, such as a provider-native web search, goes where a + // reload puts it: after the last message of a lower `seq`, before the streamed answer. + // Any other row closes the turn, a failure included, and stays last: above a streamed + // message that never got a row, it would hide the turn's failure. + let at = messages.length + for (let j = messages.length - 1; row.role === 'tool' && j >= 0; j--) { + const seq = messages[j].seq + if (seq !== undefined && seq < row.seq!) { + at = j + 1 + break + } + } + messages.splice(at, 0, row) + } + } + this.#set({ messages }) + } + + #lastSeq(): number | undefined { + let last: number | undefined + for (const m of this.#state.messages) { + if (m.seq !== undefined && (last === undefined || m.seq > last)) last = m.seq + } + return last + } + + /** Writes the current messages to local history; the conversation entry itself is `#rememberConversation`'s. */ + #persistLocal(): void { + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + this.#local.saveMessages(id, this.#state.messages) + } + + /** + * Puts the current conversation at the head of local history. Only a turn moves + * a conversation there: merely viewing one must not reorder the list. + */ + #rememberConversation(): void { + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + const conversation = this.#state.conversations.find((c) => c.id === id) + if (conversation) this.#local.upsertConversation(conversation) + } + + /** Whether an unreadable server history should silently become local history. */ + #fallBackToLocal(e: unknown): boolean { + if (this.#state.history !== 'server' || this.#config.historyExplicit) return false + if (e instanceof WindmillApiError && (e.status === 401 || e.status === 403)) { + this.#set({ history: 'local' }) + // The conversation now lives in the browser; list it there like one started local. + this.#rememberConversation() + return true + } + return false + } + + #turnActive(turn: Turn): boolean { + return this.#turn === turn && this.#state.conversationId === turn.conversationId + } + + /** Stops following the current answer; the flow itself keeps running. */ + #detachTurn(): void { + const turn = this.#turn + if (!turn) return + this.#turn = undefined + turn.controller.abort() + } + + /** + * Leaves the current conversation (for another one, or because the page goes + * away). A turn still in flight is detached and what it showed so far is kept, + * written out now rather than on the debounce that may never fire. + */ + #leaveConversation(): void { + if (this.#turn) { + this.#detachTurn() + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + } + if (this.#persistTimer) this.#persistLocal() + } + + #set(patch: Partial): void { + this.#state = { ...this.#state, ...patch } + for (const listener of this.#listeners) listener(this.#state) + if (this.#state.history === 'local' && this.#state.conversationId) { + clearTimeout(this.#persistTimer) + this.#persistTimer = setTimeout(() => this.#persistLocal(), PERSIST_DEBOUNCE_MS) + } + } +} + +function fromRow(row: FlowConversationMessage): ChatMessage { + const toolName = + row.message_type === 'tool' + ? /^Used (.+) tool$/.exec(row.content)?.[1] ?? /^Error executing (.+)$/.exec(row.content)?.[1] + : undefined + const success = row.success ?? true + return { + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + reasoning: row.reasoning ?? undefined, + attachments: row.attachments ?? undefined, + // The call the row carries: the model's arguments and what the model got back. For a + // failed tool the result is what it failed with, and the row's text names the tool + // rather than the reason. + tool: toolName + ? { + name: toolName, + status: success ? 'success' : 'error', + arguments: row.tool_arguments ?? undefined, + result: row.tool_result ?? undefined + } + : undefined + } +} + +function fromConversation(row: FlowConversation): Conversation { + return { + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at, + isTest: row.is_test + } +} + +function assistantMessage(content: string, success: boolean, jobId: string | undefined): ChatMessage { + return { + id: `local-${randomId()}`, + role: 'assistant', + content, + success, + createdAt: now(), + jobId, + pending: false + } +} + +function finalized(messages: ChatMessage[]): ChatMessage[] { + return messages.some((m) => m.pending) + ? messages.map((m) => (m.pending ? { ...m, pending: false } : m)) + : messages +} + +function toError(e: unknown): Error { + return e instanceof Error ? e : new Error(String(e)) +} diff --git a/chat-sdk/src/config.ts b/chat-sdk/src/config.ts new file mode 100644 index 0000000000..1903149d58 --- /dev/null +++ b/chat-sdk/src/config.ts @@ -0,0 +1,85 @@ +import type { ChatOptions, FetchLike, HistoryMode, StorageLike, TokenSource } from './types' + +export interface ResolvedConfig { + flowPath: string + baseUrl: string + workspace: string + token: TokenSource | undefined + history: HistoryMode + /** The caller chose `history`; a failing server history is then an error, not a fallback. */ + historyExplicit: boolean + inputs: Record + fetch: FetchLike | undefined + storage: StorageLike | undefined + storageKey: string | undefined + pageSize: number + pollDelayMs: number | undefined + run: ChatOptions['run'] + onFinish: ChatOptions['onFinish'] + onError: ChatOptions['onError'] +} + +export interface RawAppContext { + baseUrl: string + workspace: string + /** The viewer's SDK token in a sandboxed raw app; the session cookie otherwise. */ + token?: string +} + +/** + * The Windmill a raw app bundle runs in. A sandboxed app gets `window.process.env` + * from its wrapper once the viewer consented to the declared SDK scopes; an + * unsandboxed one runs on the Windmill origin with the viewer's session and only + * has `window.ctx`. + */ +export function detectRawApp(): RawAppContext | undefined { + const g = globalThis as { + process?: { env?: Record } + ctx?: { workspace?: unknown } + location?: { origin?: string } + } + const env = g.process?.env + if (env?.WM_RAW_APP === 'true' && env.WM_TOKEN && env.BASE_URL && env.WM_WORKSPACE) { + return { baseUrl: env.BASE_URL, workspace: env.WM_WORKSPACE, token: env.WM_TOKEN } + } + const workspace = g.ctx?.workspace + const origin = g.location?.origin + if (typeof workspace === 'string' && workspace && origin && origin !== 'null') { + return { baseUrl: origin, workspace } + } + return undefined +} + +export function resolveConfig(options: ChatOptions): ResolvedConfig { + if (!options.flowPath) throw new Error('windmill-chat: flowPath is required') + const explicitToken = options.token !== undefined + const detected = + options.baseUrl && options.workspace && explicitToken ? undefined : detectRawApp() + const baseUrl = options.baseUrl ?? detected?.baseUrl + const workspace = options.workspace ?? detected?.workspace + if (!baseUrl || !workspace) { + throw new Error( + 'windmill-chat: pass baseUrl and workspace. They are only detected inside a raw app: an unsandboxed one on the Windmill origin, or a sandboxed one whose policy declares frontend SDK scopes.' + ) + } + // The raw app's token belongs to its own Windmill; it never travels to another origin. + const token = + options.token ?? (options.baseUrl === undefined || options.baseUrl === detected?.baseUrl ? detected?.token : undefined) + return { + flowPath: options.flowPath, + baseUrl, + workspace, + token, + history: options.history ?? (explicitToken ? 'local' : 'server'), + historyExplicit: options.history !== undefined, + inputs: options.inputs ?? {}, + fetch: options.fetch, + storage: options.storage, + storageKey: options.storageKey, + pageSize: options.pageSize ?? 50, + pollDelayMs: options.pollDelayMs, + run: options.run, + onFinish: options.onFinish, + onError: options.onError + } +} diff --git a/chat-sdk/src/follow.ts b/chat-sdk/src/follow.ts new file mode 100644 index 0000000000..50f101393e --- /dev/null +++ b/chat-sdk/src/follow.ts @@ -0,0 +1,71 @@ +import type { WindmillChatApi } from './api' +import { createStreamEventParser, type AgentStreamEvent } from './stream' +import { abortError, sleep } from './utils' + +const RECONNECT_DELAY_MS = 300 + +export type FollowEvent = + /** Agent events decoded from the job's result stream; empty when a chunk ended mid-line. */ + | { type: 'stream'; events: AgentStreamEvent[] } + | { type: 'completed'; result: unknown } + +/** + * Follows a job to completion across the server's stream timeouts: every + * connection resumes from the last `stream_offset`, so no delta is repeated and + * the flow is never re-run. `onOffset` reports each offset, and its loss, so a + * caller can resume later from another connection (see the AI SDK transport). + * + * The offset indexes the stream of one sub-job (`flow_stream_job_id`, the flow's + * streaming step). A retried step gets a new one, so when the id changes the + * offset is dropped and the connection reopened from that sub-job's start. + */ +export async function* followJob( + api: WindmillChatApi, + jobId: string, + options: { signal?: AbortSignal; streamOffset?: number; onOffset?: (offset: number | undefined) => void } = {} +): AsyncGenerator { + let parser = createStreamEventParser() + let offset = options.streamOffset + let streamJobId: string | undefined + while (true) { + let reopen = false + for await (const update of api.streamJob(jobId, { streamOffset: offset, signal: options.signal })) { + if (update.type === 'ping') continue + if (update.type === 'timeout') { + reopen = true + break + } + if (update.type === 'error') throw new Error(update.error) + if (update.type === 'notfound') throw new Error(`Job ${jobId} not found`) + if (update.flow_stream_job_id && update.flow_stream_job_id !== streamJobId) { + const switched = streamJobId !== undefined && offset !== undefined + streamJobId = update.flow_stream_job_id + if (switched) { + // This connection skipped the new sub-job's first chunks: start it over. + offset = undefined + options.onOffset?.(undefined) + parser = createStreamEventParser() + reopen = true + break + } + } + if (update.stream_offset !== undefined) { + offset = update.stream_offset + options.onOffset?.(offset) + } + if (update.new_result_stream) { + yield { type: 'stream', events: parser.push(update.new_result_stream) } + } + if (update.completed) { + const rest = parser.flush() + if (rest.length > 0) yield { type: 'stream', events: rest } + yield { type: 'completed', result: update.only_result } + return + } + } + if (options.signal?.aborted) throw abortError() + // The server closes the connection after its timeout; a dropped connection looks + // the same minus the event. Either way the offset lets the next one resume. + if (!reopen) await sleep(RECONNECT_DELAY_MS, options.signal) + } +} diff --git a/chat-sdk/src/history.ts b/chat-sdk/src/history.ts new file mode 100644 index 0000000000..b2eb124e8f --- /dev/null +++ b/chat-sdk/src/history.ts @@ -0,0 +1,97 @@ +import type { ChatMessage, Conversation, StorageLike } from './types' + +export interface LocalHistory { + listConversations(): Conversation[] + getMessages(conversationId: string): ChatMessage[] + upsertConversation(conversation: Conversation): void + /** Changes a stored conversation's title in place; unlike `upsertConversation`, its position is kept. */ + renameConversation(conversationId: string, title: string): void + saveMessages(conversationId: string, messages: ChatMessage[]): void + deleteConversation(conversationId: string): void +} + +interface Snapshot { + v: 1 + conversations: Conversation[] + messages: Record +} + +const MAX_CONVERSATIONS = 100 + +/** + * Conversation history in the browser, for a credential shared by every visitor + * (server history would show them each other's chats). One key per flow, and per + * `storageKey` when the caller sets one; every operation re-reads the store so + * several tabs stay consistent. + */ +export function createLocalHistory(storage: StorageLike | undefined, key: string): LocalHistory { + const store = storage ?? defaultStorage() + const read = (): Snapshot => { + try { + const raw = store.getItem(key) + if (raw) { + const parsed = JSON.parse(raw) as Snapshot + if (parsed && parsed.v === 1) return parsed + } + } catch { + // unreadable: start over + } + return { v: 1, conversations: [], messages: {} } + } + const write = (snapshot: Snapshot) => { + try { + store.setItem(key, JSON.stringify(snapshot)) + } catch { + // quota or private mode: the page keeps working from memory + } + } + return { + listConversations: () => read().conversations, + getMessages: (id) => read().messages[id] ?? [], + upsertConversation(conversation) { + const s = read() + const rest = s.conversations.filter((c) => c.id !== conversation.id) + s.conversations = [conversation, ...rest] + for (const dropped of s.conversations.splice(MAX_CONVERSATIONS)) { + delete s.messages[dropped.id] + } + write(s) + }, + renameConversation(id, title) { + const s = read() + s.conversations = s.conversations.map((c) => (c.id === id ? { ...c, title } : c)) + write(s) + }, + saveMessages(id, messages) { + const s = read() + s.messages[id] = messages.map((m) => ({ ...m, pending: false })) + write(s) + }, + deleteConversation(id) { + const s = read() + s.conversations = s.conversations.filter((c) => c.id !== id) + delete s.messages[id] + write(s) + } + } +} + +function defaultStorage(): StorageLike { + try { + const ls = globalThis.localStorage + if (ls) { + const probe = '__windmill_chat_probe__' + ls.setItem(probe, '1') + ls.removeItem(probe) + return ls + } + } catch { + // no localStorage (SSR, blocked storage): fall through + } + const memory = new Map() + return { + getItem: (k) => memory.get(k) ?? null, + setItem: (k, v) => void memory.set(k, v), + removeItem: (k) => void memory.delete(k) + } +} diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts new file mode 100644 index 0000000000..9ed2ed0b7e --- /dev/null +++ b/chat-sdk/src/index.ts @@ -0,0 +1,35 @@ +export { createChat } from './chat' +export { detectRawApp, type RawAppContext } from './config' +export { + WindmillChatApi, + WindmillApiError, + readServerSentEvents, + type WindmillChatApiOptions, + type ConversationKind, + type FlowConversation, + type FlowConversationMessage, + type JobUpdateEvent, + type CompletedJobResult +} from './api' +export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream' +export { followJob, type FollowEvent } from './follow' +export { extractChatAnswer, conversationIdFor } from './utils' +export { storedAttachmentName, uploadAttachments, CHAT_UPLOADS_PREFIX, type UploadedAttachment } from './attachments' +export type { + AttachmentsInput, + AttachmentUpload, + Chat, + ChatAttachment, + ChatMessage, + ChatOptions, + ChatRole, + ChatState, + ChatStatus, + Conversation, + FetchLike, + HistoryMode, + SendMessageOptions, + StorageLike, + TokenSource, + ToolInvocation +} from './types' diff --git a/chat-sdk/src/react.ts b/chat-sdk/src/react.ts new file mode 100644 index 0000000000..33193842f2 --- /dev/null +++ b/chat-sdk/src/react.ts @@ -0,0 +1,74 @@ +import { useEffect, useMemo, useRef, useSyncExternalStore } from 'react' +import { createChat } from './chat' +import type { Chat, ChatOptions, ChatState } from './types' + +export type UseWindmillChat = ChatState & + Pick< + Chat, + | 'sendMessage' + | 'stop' + | 'newConversation' + | 'selectConversation' + | 'loadConversations' + | 'deleteConversation' + | 'renameConversation' + | 'loadOlderMessages' + > & { chat: Chat } + +/** + * A chat on a chat-mode flow. The chat is created once per `flowPath`, `baseUrl`, + * `workspace`, `history`, `storageKey`, credential and presence of `run`, and + * destroyed on unmount. A credential change is a new user, whose chat must not + * carry the previous one's state: a different token string, or a switch between no + * token, a token string and a token function, all recreate it. A token function is + * read through a ref on every call, so a new closure per render changes what the + * next call runs and nothing else; pass a `storageKey` per user when local history + * must not be shared. `run`, the callbacks and `inputs` are read the same way: the + * latest render's values go with the next message. + */ +export function useWindmillChat(options: ChatOptions): UseWindmillChat { + const latest = useRef(options) + latest.current = options + const credential = + typeof options.token === 'function' ? 'fn' : typeof options.token === 'string' ? `str:${options.token}` : 'none' + // A custom runner replaces the deployed flow call, so its presence is part of what the chat is. + const customRun = options.run !== undefined + const chat = useMemo( + () => + createChat({ + ...options, + // Sent per message from the latest render instead, so a removed key stays removed. + inputs: undefined, + token: + typeof options.token === 'function' + ? () => { + const token = latest.current.token + return typeof token === 'function' ? token() : (token ?? '') + } + : options.token, + run: customRun ? (args, turn) => (latest.current.run ?? options.run!)(args, turn) : undefined, + onFinish: (turn) => latest.current.onFinish?.(turn), + onError: (error, turn) => latest.current.onError?.(error, turn) + }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [options.flowPath, options.baseUrl, options.workspace, options.history, options.storageKey, credential, customRun] + ) + useEffect(() => () => chat.destroy(), [chat]) + const state = useSyncExternalStore(chat.subscribe, chat.getState, chat.getState) + return useMemo( + () => ({ + ...state, + chat, + sendMessage: (text, options) => + chat.sendMessage(text, { ...options, inputs: { ...latest.current.inputs, ...options?.inputs } }), + stop: chat.stop, + newConversation: chat.newConversation, + selectConversation: chat.selectConversation, + loadConversations: chat.loadConversations, + deleteConversation: chat.deleteConversation, + renameConversation: chat.renameConversation, + loadOlderMessages: chat.loadOlderMessages + }), + [state, chat] + ) +} diff --git a/chat-sdk/src/stream.ts b/chat-sdk/src/stream.ts new file mode 100644 index 0000000000..c6969bb7b4 --- /dev/null +++ b/chat-sdk/src/stream.ts @@ -0,0 +1,65 @@ +/** The events an AI agent step streams, one JSON object per line of the job's result stream. */ +export type AgentStreamEvent = + | { type: 'token_delta'; content: string } + | { type: 'reasoning_token_delta'; content: string } + | { type: 'tool_call'; call_id: string; function_name: string } + | { type: 'tool_call_arguments'; call_id: string; function_name: string; arguments: string } + | { type: 'tool_execution'; call_id: string; function_name: string } + | { + type: 'tool_result' + call_id: string + function_name: string + result: string + success: boolean + } + +const KNOWN_TYPES = new Set([ + 'token_delta', + 'reasoning_token_delta', + 'tool_call', + 'tool_call_arguments', + 'tool_execution', + 'tool_result' +]) + +/** + * Incremental parser for the `new_result_stream` chunks of a job update. A chunk is + * not guaranteed to end on a line boundary, so an incomplete last line waits for the + * next `push` (or `flush` once the job completes). + */ +export function createStreamEventParser() { + let pending = '' + return { + push(chunk: string): AgentStreamEvent[] { + pending += chunk + const lastNewline = pending.lastIndexOf('\n') + if (lastNewline === -1) return [] + const complete = pending.slice(0, lastNewline) + pending = pending.slice(lastNewline + 1) + return parseStreamEvents(complete) + }, + flush(): AgentStreamEvent[] { + const rest = pending + pending = '' + return parseStreamEvents(rest) + } + } +} + +/** Parses complete NDJSON lines; lines that aren't agent events are skipped. */ +export function parseStreamEvents(ndjson: string): AgentStreamEvent[] { + const events: AgentStreamEvent[] = [] + for (const line of ndjson.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + try { + const parsed = JSON.parse(trimmed) + if (parsed && typeof parsed === 'object' && KNOWN_TYPES.has(parsed.type)) { + events.push(parsed as AgentStreamEvent) + } + } catch { + // not an agent event + } + } + return events +} diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts new file mode 100644 index 0000000000..99e16fe304 --- /dev/null +++ b/chat-sdk/src/types.ts @@ -0,0 +1,188 @@ +export type ChatRole = 'user' | 'assistant' | 'tool' | 'system' + +/** + * - `idle`: ready for a message + * - `submitted`: the message was sent, no answer has started streaming yet + * - `streaming`: the answer is arriving + * - `error`: the last turn failed; `error` holds why. Sending again is allowed. + */ +export type ChatStatus = 'idle' | 'submitted' | 'streaming' | 'error' + +/** + * Where conversation history lives. + * - `server`: Windmill's conversation store. Each Windmill user only sees their own + * conversations, so use it with the viewer's own session or a per-user token. + * - `local`: the browser's storage. Right for a token shared by every visitor. + * - `none`: nothing is kept beyond the current page. + */ +export type HistoryMode = 'server' | 'local' | 'none' + +export interface ToolInvocation { + callId?: string + name: string + /** The arguments the model passed, as a JSON string. */ + arguments?: string + result?: string + status: 'running' | 'success' | 'error' +} + +export interface ChatAttachment { input: string; s3: string; storage?: string; filename?: string } + +export interface ChatMessage { + id: string + role: ChatRole + content: string + /** The model's reasoning summary, when the provider streams one. */ + reasoning?: string + /** The call on a `tool` message, from the live stream or its stored row; `callId` is only known from the stream. */ + tool?: ToolInvocation + success: boolean + createdAt: string + jobId?: string + /** The flow step that produced the message. */ + stepName?: string + /** The files a user message carried, as object-storage references. */ + attachments?: ChatAttachment[] + /** True while the message is optimistic or still streaming. */ + pending: boolean + /** Id of the persisted row once the server has it; `id` itself never changes, so list keys stay stable. */ + serverId?: string + /** The server's cursor for a persisted message; unset for one created on the client. */ + seq?: number +} + +export interface Conversation { + id: string + title: string | undefined + createdAt: string + updatedAt: string + /** + * Started from the flow editor's test panel rather than a deployed run. Known once the + * server has listed the conversation; unset for one only this client has seen. + */ + isTest?: boolean +} + +export interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: ChatStatus + error: Error | undefined + conversations: Conversation[] + /** Where history is read from. Starts as configured; drops from `server` to `local` when the credential cannot read conversations. */ + history: HistoryMode + loadingMessages: boolean + hasMoreMessages: boolean +} + +export type TokenSource = string | (() => string | Promise) + +export type StorageLike = Pick + +export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise + +export interface ChatOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Windmill origin, e.g. `https://app.windmill.dev`. Detected inside a raw app. */ + baseUrl?: string + /** Detected inside a raw app. */ + workspace?: string + /** + * A Windmill token, or a function returning one (called before every request, so it + * can fetch a short-lived token from your backend). Omit it to use the viewer's + * session: the cookie on the Windmill origin, or a sandboxed raw app's SDK token. + */ + token?: TokenSource + /** Defaults to `server` with the viewer's session and `local` with an explicit token. */ + history?: HistoryMode + /** Extra flow inputs sent with every message, next to `user_message`. */ + inputs?: Record + fetch?: FetchLike + /** Backing store for `local` history. Defaults to `localStorage`. */ + storage?: StorageLike + /** + * Namespace for `local` history, e.g. the signed-in user's id. Local history is + * per browser, per flow; without this, users sharing a browser share it. + */ + storageKey?: string + /** Messages fetched per page of server history. */ + pageSize?: number + /** + * How often, in milliseconds, the server polls a running turn for the stream + * (Enterprise; 50 at the fastest, other servers ignore it). Unset, the server + * relaxes from 100 ms to 3 s over a long turn. + */ + pollDelayMs?: number + /** + * Runs the flow for a turn and returns the job id, instead of the deployed flow at + * `flowPath`. `args` carries `user_message` and the extra inputs; the run must set + * `memory_id` to the conversation id for the conversation and its memory to line up. + * Windmill's own editor uses this to chat with an undeployed flow through a preview run. + */ + run?: (args: Record, turn: { conversationId: string; signal: AbortSignal }) => Promise + /** Called once a turn has its answer (a failed flow included: its error is the answer). */ + onFinish?: (turn: { conversationId: string; jobId?: string; messages: ChatMessage[] }) => void + /** Called when a turn could not run or be followed; `state.error` holds the same error. */ + onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void +} + +/** A file sent with a message. It is uploaded to the workspace's object storage before the run starts. */ +export interface AttachmentUpload { + /** Kept as the last segment of the stored key, its extension corrected to the media type for PNG, JPEG and PDF. */ + name: string + /** The bytes: a Blob, or a `data:` URL of them. */ + data: Blob | string + /** The file's media type. Defaults to the Blob's own type, or the data URL's. */ + mediaType?: string +} + +/** The flow input the uploaded attachments are handed to: an `s3object` (`multiple: false`) or an `s3object[]`. */ +export interface AttachmentsInput { + name: string + multiple: boolean +} + +export interface SendMessageOptions { + /** Extra flow inputs for this message, on top of `ChatOptions.inputs`. */ + inputs?: Record + /** + * Files to upload and hand to the flow as `{ s3, filename }` objects in `attachmentsInput`, + * the way an AI agent step reads `user_attachments`. A failed upload rejects `sendMessage` + * and the run never starts; `stop()` during the upload does the same with an `AbortError`. + */ + attachments?: AttachmentUpload[] + /** Required with `attachments`, which also need message text. With `multiple: false`, more than one attachment is refused before anything uploads. */ + attachmentsInput?: AttachmentsInput +} + +export interface Chat { + getState(): ChatState + /** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */ + subscribe(listener: (state: ChatState) => void): () => void + /** + * Sends a message in the current conversation, starting one when there is none. Resolves + * when the answer is complete. Rejects when the message could not be sent at all — a turn + * already running, an attachment that failed to upload — without touching the transcript. + */ + sendMessage(text: string, options?: SendMessageOptions): Promise + /** Stops following the answer and asks Windmill to cancel the run. */ + stop(): Promise + newConversation(): void + selectConversation(conversationId: string): Promise + /** + * `kind` narrows server history to the flow editor's test chats, the deployed flow's + * own (the server's default), or both. Local history has no test chats and ignores it. + */ + loadConversations(options?: { + page?: number + perPage?: number + kind?: 'test' | 'deployed' | 'all' + }): Promise + deleteConversation(conversationId: string): Promise + /** Sets a conversation's title. The list keeps its order: only a turn moves a conversation. */ + renameConversation(conversationId: string, title: string): Promise + loadOlderMessages(): Promise + /** Stops background work (stream, polling) and writes local history out. The chat stays usable. */ + destroy(): void +} diff --git a/chat-sdk/src/utils.ts b/chat-sdk/src/utils.ts new file mode 100644 index 0000000000..7973914e93 --- /dev/null +++ b/chat-sdk/src/utils.ts @@ -0,0 +1,140 @@ +export function randomId(): string { + const c = globalThis.crypto + if (c?.randomUUID) return c.randomUUID() + // `randomUUID` needs a secure context; a plain http dev origin has `getRandomValues` only. + const bytes = new Uint8Array(16) + c.getRandomValues(bytes) + return formatUuid(bytes, 4) +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +export function isUuid(value: string): boolean { + return UUID_RE.test(value) +} + +/** + * The Windmill conversation id for an arbitrary chat id. A UUID is used as is; + * anything else (an AI SDK chat id, for instance) maps to the same UUID every time, + * so a page can reopen its conversation without storing a second id. Hashed in plain + * JS: `crypto.subtle` only exists in secure contexts, and the mapping must not depend + * on the origin's scheme. + */ +export function conversationIdFor(chatId: string): string { + if (isUuid(chatId)) return chatId.toLowerCase() + const bytes = new TextEncoder().encode(`windmill-chat:${chatId}`) + const out = new Uint8Array(16) + for (const [i, seed] of [0xcbf29ce484222325n, 0x84222325cbf29ce4n].entries()) { + let h = fnv1a64(bytes, seed) + for (let b = 7; b >= 0; b--) { + out[i * 8 + b] = Number(h & 0xffn) + h >>= 8n + } + } + return formatUuid(out, 5) +} + +function fnv1a64(bytes: Uint8Array, seed: bigint): bigint { + let h = seed + for (const byte of bytes) { + h ^= BigInt(byte) + h = (h * 0x100000001b3n) & 0xffffffffffffffffn + } + return h +} + +function formatUuid(bytes: Uint8Array, version: 4 | 5): string { + bytes[6] = (bytes[6] & 0x0f) | (version << 4) + bytes[8] = (bytes[8] & 0x3f) | 0x80 + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +export function parseJsonOr(text: string | undefined): unknown { + if (text === undefined) return undefined + try { + return JSON.parse(text) + } catch { + return text + } +} + +export function now(): string { + return new Date().toISOString() +} + +/** Same rule as the server: the first message, cut to 25 characters. */ +export function conversationTitle(firstMessage: string): string { + const chars = Array.from(firstMessage) + return chars.length > 25 ? `${chars.slice(0, 25).join('')}...` : firstMessage +} + +/** The server's bound on a typed title: 252 characters plus an ellipsis fits its 255-char column. */ +export function truncateTitle(title: string): string { + const chars = Array.from(title) + return chars.length > 252 ? `${chars.slice(0, 252).join('')}...` : title +} + +export function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(abortError()) + const onAbort = () => { + clearTimeout(timer) + reject(abortError()) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +export function abortError(): Error { + return new DOMException('The operation was aborted', 'AbortError') +} + +export function isAbortError(e: unknown): boolean { + return e instanceof Error && e.name === 'AbortError' +} + +/** + * The text a chat shows for a flow result, following what Windmill persists as the + * assistant message when the last step is not an AI agent: `windmill_chat_answer` + * when the result carries one (null means no message), an agent result's `output`, + * a string as is, anything else as JSON. + */ +export function extractChatAnswer(result: unknown): string | undefined { + if (result === null || result === undefined) return undefined + if (typeof result === 'string') return result + if (typeof result === 'object' && !Array.isArray(result)) { + const obj = result as Record + if ('windmill_chat_answer' in obj) return formatAnswer(obj.windmill_chat_answer) + if ('output' in obj && Array.isArray(obj.messages)) return formatAnswer(obj.output) + } + return JSON.stringify(result, null, 2) +} + +function formatAnswer(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined + return typeof value === 'string' ? value : JSON.stringify(value, null, 2) +} + +/** A completed job whose result is Windmill's error envelope. */ +export function isErrorResult(result: unknown): result is { error: Record } { + return ( + typeof result === 'object' && + result !== null && + 'error' in result && + typeof (result as { error: unknown }).error === 'object' && + (result as { error: unknown }).error !== null + ) +} + +export function errorResultMessage(result: { error: Record }): string { + const { message, name } = result.error + if (typeof message === 'string' && message) { + return typeof name === 'string' && name && name !== 'Error' ? `${name}: ${message}` : message + } + return JSON.stringify(result.error, null, 2) +} diff --git a/chat-sdk/test/ai-sdk-chat.test.ts b/chat-sdk/test/ai-sdk-chat.test.ts new file mode 100644 index 0000000000..a0bbb7b8a1 --- /dev/null +++ b/chat-sdk/test/ai-sdk-chat.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from 'bun:test' +import { Chat } from '@ai-sdk/react' +import { createWindmillChatTransport } from '../src/ai-sdk' +import { fetchMock, json, ndjson, sse, text } from './support' + +const FLOW = 'f/chat/agent' + +/** The AI SDK's own chunk processor consuming the transport, as `useChat` would. */ +describe('AI SDK Chat over the Windmill transport', () => { + test('builds the assistant message parts and settles to ready', async () => { + let jobs = 0 + const { fetch, calls } = fetchMock( + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text(`job-${++jobs}`) : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'reasoning_token_delta', content: 'so ' }, + { type: 'token_delta', content: 'The answer ' }, + { type: 'token_delta', content: 'is 42' } + ), + stream_offset: 6, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { error: { message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/completed/get_result_maybe/job-2') ? json({ completed: true, success: false }) : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, token: 'tok', fetch }) + const chat = new Chat({ id: 'e2e-chat', transport }) + + await chat.sendMessage({ text: 'what is it?' }) + + expect(chat.status).toBe('ready') + expect(chat.messages.map((m) => m.role)).toEqual(['user', 'assistant']) + const parts = chat.messages[1].parts + expect(parts.map((p) => p.type)).toEqual(['dynamic-tool', 'reasoning', 'text']) + expect(parts[0]).toMatchObject({ toolName: 'lookup', toolCallId: 'c1', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(parts[1]).toMatchObject({ type: 'reasoning', text: 'so ', state: 'done' }) + expect(parts[2]).toMatchObject({ type: 'text', text: 'The answer is 42', state: 'done' }) + // Both turns of the chat ran in the same Windmill conversation. + const memoryIds = calls.filter((c) => c.method === 'POST').map((c) => c.url.searchParams.get('memory_id')) + expect(memoryIds[0]).toBe(transport.conversationId('e2e-chat')) + + await chat.sendMessage({ text: 'and now fail' }) + expect(chat.status).toBe('error') + expect(chat.error?.message).toBe('boom') + expect(memoryIds.length === 1 || calls.filter((c) => c.method === 'POST')[1].url.searchParams.get('memory_id') === memoryIds[0]).toBe(true) + }) +}) diff --git a/chat-sdk/test/ai-sdk.test.ts b/chat-sdk/test/ai-sdk.test.ts new file mode 100644 index 0000000000..70b7c40f33 --- /dev/null +++ b/chat-sdk/test/ai-sdk.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, test } from 'bun:test' +import type { UIMessage, UIMessageChunk } from 'ai' +import { createWindmillChatTransport, toUIMessages } from '../src/ai-sdk' +import type { ChatMessage } from '../src/types' +import { fetchMock, json, messageRow, ndjson, sse, text, type Route } from './support' + +const FLOW = 'f/chat/agent' +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +const userMessage = (text: string): UIMessage => ({ id: 'u1', role: 'user', parts: [{ type: 'text', text }] }) + +async function collect(stream: ReadableStream): Promise { + const chunks: UIMessageChunk[] = [] + const reader = stream.getReader() + while (true) { + const { value, done } = await reader.read() + if (done) return chunks + chunks.push(value) + } +} + +describe('createWindmillChatTransport', () => { + test('maps an agent turn to AI SDK chunks and derives the conversation from the chat id', async () => { + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'think' }, + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_execution', call_id: 'c1', function_name: 'lookup' } + ), + stream_offset: 5 + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '{"answer":42}', success: true }, + { type: 'token_delta', content: '42' } + ), + stream_offset: 7, + completed: true, + only_result: { output: '42', messages: [] } + } + ]) + : undefined + ) + const transport = createWindmillChatTransport({ + baseUrl: 'http://wm.test', + workspace: 'ws', + flowPath: FLOW, + token: 'tok', + inputs: { tone: 'kind' }, + fetch + }) + const chunks = await collect( + await transport.sendMessages({ + trigger: 'submit-message', + chatId: 'chat-abc', + messageId: undefined, + messages: [userMessage('what is it?')], + abortSignal: undefined, + body: { locale: 'fr' } + }) + ) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.body).toEqual({ tone: 'kind', locale: 'fr', user_message: 'what is it?' }) + expect(runCall.url.searchParams.get('memory_id')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abc')).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) + expect(transport.conversationId('chat-abc')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abd')).not.toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('4D6E5C8C-2C3B-4E1A-9F31-0B2E6F1C9D10')).toBe( + '4d6e5c8c-2c3b-4e1a-9f31-0b2e6f1c9d10' + ) + + const shape = chunks.map((c) => ('delta' in c ? `${c.type}:${c.delta}` : c.type)) + expect(shape).toEqual([ + 'start', + 'reasoning-start', + 'reasoning-delta:think', + 'text-start', + 'text-delta:Let me ', + 'reasoning-end', + 'text-end', + 'tool-input-start', + 'tool-input-available', + 'tool-output-available', + 'text-start', + 'text-delta:42', + 'text-end', + 'finish' + ]) + expect(chunks.find((c) => c.type === 'tool-input-available')).toMatchObject({ + toolCallId: 'c1', + toolName: 'lookup', + input: { q: 1 }, + dynamic: true + }) + expect(chunks.find((c) => c.type === 'tool-output-available')).toMatchObject({ output: { answer: 42 } }) + // The job finished, so there is nothing to reconnect to. + expect(await transport.reconnectToStream({ chatId: 'chat-abc' })).toBeNull() + }) + + test('answers from the flow result when nothing streamed, and reports a failed flow as an error', async () => { + let turn = 0 + const { fetch } = fetchMock( + run, + (c) => { + if (c.url.pathname !== streamPath) return undefined + turn++ + return sse([ + turn === 1 + ? { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } } + : { type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } } + ]) + }, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false }) + : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch }) + const send = () => + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [userMessage('hi')], + abortSignal: undefined + }) + const first = await collect(await send()) + expect(first.map((c) => ('delta' in c ? c.delta : c.type))).toEqual(['start', 'text-start', 'From a script', 'text-end', 'finish']) + const second = await collect(await send()) + expect(second.map((c) => c.type)).toEqual(['start', 'error']) + expect(second[1]).toMatchObject({ errorText: 'ExecutionErr: boom' }) + }) + + test('loads the attachments of a user row, the call an MCP tool row carries and the reasoning behind an answer', async () => { + const { fetch } = fetchMock((c) => + c.method === 'GET' && c.url.pathname.endsWith('/messages') + ? json([ + messageRow(1, 'user', 'hi', { attachments: [{ input: 'files', s3: 'chat/a.png', filename: 'a.png' }] }), + messageRow(2, 'tool', 'Used lookup tool', { job_id: 'agent-job', reasoning: 'why', tool_arguments: '{"q":1}', tool_result: '42' }), + messageRow(3, 'assistant', 'The answer is 42', { reasoning: 'hmm' }) + ]) + : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch }) + const ui = await transport.loadMessages('c') + expect(ui.map((m) => m.parts.map((p) => p.type))).toEqual([['text'], ['reasoning', 'dynamic-tool', 'reasoning', 'text']]) + expect(ui[0].metadata).toEqual({ attachments: [{ input: 'files', s3: 'chat/a.png', filename: 'a.png' }] }) + expect(ui[1].metadata).toBeUndefined() + expect(ui[1].parts[0]).toMatchObject({ type: 'reasoning', text: 'why' }) + expect(ui[1].parts[1]).toMatchObject({ toolName: 'lookup', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(ui[1].parts[2]).toMatchObject({ type: 'reasoning', text: 'hmm' }) + }) + + test('refuses attachments with a clear error', async () => { + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch: fetchMock().fetch }) + await expect( + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [{ id: 'u', role: 'user', parts: [{ type: 'file', mediaType: 'image/png', url: 'data:...' }] }], + abortSignal: undefined + }) + ).rejects.toThrow('attachments are not supported') + }) +}) + +describe('toUIMessages', () => { + test('folds a turn into one assistant message with reasoning, tool and text parts', () => { + const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'thanks' }, + { ...base, id: 't2', role: 'tool', content: 'Error executing lookup', success: false, tool: { name: 'lookup', status: 'error' } } + ] + const ui = toUIMessages(messages) + expect(ui.map((m) => [m.id, m.role, m.parts.map((p) => p.type)])).toEqual([ + ['u1', 'user', ['text']], + ['t1', 'assistant', ['dynamic-tool', 'reasoning', 'text']], + ['u2', 'user', ['text']], + ['t2', 'assistant', ['dynamic-tool']] + ]) + expect(ui[1].parts[0]).toMatchObject({ toolCallId: 'c1', toolName: 'lookup', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(ui[3].parts[0]).toMatchObject({ state: 'output-error', errorText: 'Error executing lookup' }) + }) + + test('keeps a stored JSON null result rather than the row text', () => { + const ui = toUIMessages([ + { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false, id: 't1', role: 'tool', content: 'Used notify tool', tool: { callId: 'c1', name: 'notify', status: 'success', arguments: '{}', result: 'null' } } + ]) + expect(ui[0].parts[0]).toMatchObject({ type: 'dynamic-tool', state: 'output-available', output: null }) + }) +}) diff --git a/chat-sdk/test/api.test.ts b/chat-sdk/test/api.test.ts new file mode 100644 index 0000000000..0facf03e97 --- /dev/null +++ b/chat-sdk/test/api.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from 'bun:test' +import { WindmillChatApi } from '../src/api' + +describe('WindmillChatApi.attachmentUrl', () => { + test('points at the workspace download endpoint, with the storage only when there is one', () => { + const api = new WindmillChatApi({ baseUrl: 'https://wm.test/api/', workspace: 'my ws' }) + expect(api.attachmentUrl({ input: 'files', s3: 'chat/a b&c.png', storage: 'secondary' })).toBe( + 'https://wm.test/api/w/my%20ws/job_helpers/download_s3_file?file_key=chat%2Fa+b%26c.png&storage=secondary' + ) + expect(api.attachmentUrl({ input: 'files', s3: 'chat/a.png' })).toBe( + 'https://wm.test/api/w/my%20ws/job_helpers/download_s3_file?file_key=chat%2Fa.png' + ) + }) +}) diff --git a/chat-sdk/test/assistant-ui.test.ts b/chat-sdk/test/assistant-ui.test.ts new file mode 100644 index 0000000000..8601ad221a --- /dev/null +++ b/chat-sdk/test/assistant-ui.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from 'bun:test' +import { groupTurns, toThreadMessage } from '../src/assistant-ui' +import type { ChatMessage } from '../src/types' + +const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + +describe('assistant-ui conversion', () => { + test('groups rows into turns and renders tool calls as content parts', () => { + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'again' }, + { ...base, id: 'a2', role: 'assistant', content: 'partial', pending: true } + ] + const turns = groupTurns(messages) + expect(turns.map((t) => [t.id, t.role, t.messages.length])).toEqual([ + ['u1', 'user', 1], + ['t1', 'assistant', 2], + ['u2', 'user', 1], + ['a2', 'assistant', 1] + ]) + const answer = toThreadMessage(turns[1]) + expect(answer).toMatchObject({ id: 't1', role: 'assistant', status: { type: 'complete', reason: 'stop' } }) + expect(answer.content).toEqual([ + { type: 'tool-call', toolCallId: 'c1', toolName: 'lookup', args: { q: 1 }, argsText: '{"q":1}', result: 42, isError: false }, + { type: 'reasoning', text: 'hmm' }, + { type: 'text', text: 'The answer is 42' } + ]) + expect(toThreadMessage(turns[3]).status).toEqual({ type: 'running' }) + expect(toThreadMessage(turns[0])).toMatchObject({ role: 'user', content: [{ type: 'text', text: 'hi' }] }) + }) + + test('keeps a stored JSON null result rather than the row text', () => { + const turn = groupTurns([ + { ...base, id: 't1', role: 'tool', content: 'Used notify tool', tool: { callId: 'c1', name: 'notify', status: 'success', arguments: '{}', result: 'null' } } + ])[0] + expect(toThreadMessage(turn).content).toMatchObject([{ type: 'tool-call', toolName: 'notify', result: null }]) + }) + + test('marks a failed answer as incomplete', () => { + const [turn] = groupTurns([{ ...base, id: 'a', role: 'assistant', content: 'boom', success: false }]) + expect(toThreadMessage(turn).status).toEqual({ type: 'incomplete', reason: 'error', error: 'boom' }) + }) +}) diff --git a/chat-sdk/test/attachments.test.ts b/chat-sdk/test/attachments.test.ts new file mode 100644 index 0000000000..9c2f577567 --- /dev/null +++ b/chat-sdk/test/attachments.test.ts @@ -0,0 +1,444 @@ +import { describe, expect, test } from 'bun:test' +import { WindmillChatApi } from '../src/api' +import { storedAttachmentName, uploadAttachments } from '../src/attachments' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { abortError } from '../src/utils' +import { fetchMock, json, memoryStorage, sse, text, type RecordedCall, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' +const UPLOAD_PATH = '/api/w/ws/job_helpers/upload_s3_file' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? text('job-1') + : undefined + +/** Stores under the key it was asked to, like the server with a `file_key`. */ +const upload: Route = (c) => + c.url.pathname === UPLOAD_PATH + ? json({ file_key: c.url.searchParams.get('file_key') }) + : undefined + +const answer: Route = (c) => + c.url.pathname === '/api/w/ws/jobs_u/getupdate_sse/job-1' + ? sse([ + { + type: 'update', + completed: true, + only_result: { output: 'ok', messages: [] } + } + ]) + : undefined + +function options(fetch: ChatOptions['fetch']): ChatOptions { + return { + flowPath: FLOW, + baseUrl: BASE, + workspace: 'ws', + token: 'tok', + fetch, + storage: memoryStorage() + } +} + +const uploads = (calls: RecordedCall[]) => calls.filter((c) => c.url.pathname === UPLOAD_PATH) +const runs = (calls: RecordedCall[]) => + calls.filter((c) => c.url.pathname.startsWith('/api/w/ws/jobs/run/')) + +const png = new Blob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], { + type: 'image/png' +}) +const pdf = new Blob(['%PDF-1.7'], { type: 'application/pdf' }) + +describe('storedAttachmentName', () => { + // The worker reads the media type from the key's extension, so it has to match the bytes. + test('renames a re-encoded image and gives a bare name its extension', () => { + expect(storedAttachmentName('photo.webp', 'image/png')).toBe('photo.png') + expect(storedAttachmentName('holiday.png', 'image/jpeg')).toBe('holiday.jpg') + expect(storedAttachmentName('contract', 'application/pdf')).toBe('contract.pdf') + expect(storedAttachmentName('report.2026.final.webp', 'image/png')).toBe( + 'report.2026.final.png' + ) + }) + + test('leaves a type it does not know alone', () => { + expect(storedAttachmentName('notes.csv', 'text/csv')).toBe('notes.csv') + }) +}) + +describe('sendMessage with attachments', () => { + test('uploads each file under the turn prefix and hands the list to the input', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + + await chat.sendMessage('read these', { + inputs: { locale: 'fr' }, + attachments: [ + { name: 'photo.webp', data: png }, + { name: 'contract', data: pdf }, + // A data URL is decoded to its bytes; the mediaType names what they are. + { + name: 'photo.webp', + data: `data:image/png;base64,${btoa('\x89PNG')}` + } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + + const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + expect(keys).toHaveLength(3) + const prefix = keys[0].split('/').slice(0, 3).join('/') + expect(prefix).toMatch(/^windmill_uploads\/chat\/[0-9a-f-]{36}$/) + expect(keys).toEqual([ + `${prefix}/0/photo.png`, + `${prefix}/1/contract.pdf`, + `${prefix}/2/photo.png` + ]) + expect(uploads(calls).map((c) => c.url.searchParams.get('content_type'))).toEqual([ + 'image/png', + 'application/pdf', + 'image/png' + ]) + expect(uploads(calls).map((c) => c.headers['content-type'])).toEqual([ + 'image/png', + 'application/pdf', + 'image/png' + ]) + expect(new Uint8Array(await uploads(calls)[2].raw!.arrayBuffer())).toEqual( + new Uint8Array([0x89, 0x50, 0x4e, 0x47]) + ) + + expect(runs(calls)[0].body).toEqual({ + locale: 'fr', + user_message: 'read these', + files: [ + { s3: `${prefix}/0/photo.png`, filename: 'photo.png' }, + { s3: `${prefix}/1/contract.pdf`, filename: 'contract.pdf' }, + { s3: `${prefix}/2/photo.png`, filename: 'photo.png' } + ] + }) + expect(chat.getState().status).toBe('idle') + }) + + test('hands a single object to an input that holds one file', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'file', multiple: false } + }) + const body = runs(calls)[0].body as Record + expect(body.file).toEqual({ + s3: expect.stringMatching(/\/0\/contract\.pdf$/), + filename: 'contract.pdf' + }) + }) + + test('refuses several files for an input that holds one, before uploading any', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('read these', { + attachments: [ + { name: 'a.pdf', data: pdf }, + { name: 'b.png', data: png } + ], + attachmentsInput: { name: 'file', multiple: false } + }) + ).rejects.toThrow('holds one file') + expect(calls).toHaveLength(0) + expect(chat.getState().messages).toEqual([]) + }) + + test('refuses attachments without message text, before uploading', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage(' ', { + attachments: [{ name: 'a.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('need a message') + expect(calls).toHaveLength(0) + }) + + test('refuses attachments without an input to put them in', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('hi', { attachments: [{ name: 'a.pdf', data: pdf }] }) + ).rejects.toThrow('attachmentsInput') + expect(calls).toHaveLength(0) + }) + + test('a failed upload rejects without a run, and withdraws the message', async () => { + const { fetch, calls } = fetchMock( + (c) => (c.url.pathname === UPLOAD_PATH ? text('no object storage', 500) : undefined), + run, + answer + ) + const chat = createChat(options(fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await expect( + chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('no object storage') + + expect(runs(calls)).toHaveLength(0) + // Shown as submitted while uploading, then withdrawn whole: no message, no conversation. + expect(statuses).toContain('submitted') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages).toEqual([]) + expect(state.conversationId).toBeUndefined() + expect(state.conversations).toEqual([]) + // The chat is free for the next message. + await chat.sendMessage('plain') + expect(runs(calls)).toHaveLength(1) + }) + + test('stop() during the upload aborts it and withdraws the message', async () => { + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + run, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState().status).toBe('submitted') + await chat.stop() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(chat.getState()).toMatchObject({ + status: 'idle', + messages: [], + conversationId: undefined + }) + }) + + test('a switch away mid-upload leaves no conversation behind', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + run, + answer + ) + const chat = createChat({ ...options(fetch), storage }) + const sending = chat.sendMessage('never runs', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + const opened = chat.getState().conversationId! + chat.newConversation() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + + expect(runs(calls)).toHaveLength(0) + expect(chat.getState().conversations.map((c) => c.id)).not.toContain(opened) + const reloaded = createChat({ ...options(fetch), storage }) + expect((await reloaded.loadConversations()).map((c) => c.id)).not.toContain(opened) + }) + + test('a failed upload aborts the rest of its batch and deletes nothing', async () => { + let first: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + (c) => { + if (c.url.pathname !== UPLOAD_PATH) return undefined + const key = c.url.searchParams.get('file_key')! + // The first file lands after the second has already failed. + if (key.includes('/0/')) return new Promise((resolve) => (first = resolve)) + setTimeout(() => first(json({ file_key: keys()[0] })), 5) + return text('quota exceeded', 507) + }, + run, + answer + ) + const keys = () => uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('read these', { + attachments: [ + { name: 'a.pdf', data: pdf }, + { name: 'b.png', data: png } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('quota exceeded') + // The upload still in flight when the other failed was told to stop. + expect(uploads(calls)[0].signal?.aborted).toBe(true) + expect(calls.filter((c) => c.method === 'DELETE')).toEqual([]) + expect(runs(calls)).toHaveLength(0) + }) + + test('a send made right after stop() is not reset by the stopped upload', async () => { + let releaseRun: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? new Promise((resolve) => (releaseRun = resolve)) + : undefined, + answer + ) + const chat = createChat(options(fetch)) + // An existing conversation, so the stopped turn and the next one share it. + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 0)) + releaseRun(text('job-1')) + await first + const stopped = chat.sendMessage('with a file', { + attachments: [{ name: 'a.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + void chat.stop() + const next = chat.sendMessage('right after') + await expect(stopped).rejects.toMatchObject({ name: 'AbortError' }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState().status).toBe('submitted') + expect(chat.getState().messages.map((m) => m.content)).toContain('right after') + expect(chat.getState().messages.map((m) => m.content)).not.toContain('with a file') + releaseRun(text('job-1')) + await next + expect(runs(calls)).toHaveLength(2) + }) + + test('a conversation is listed only once its run starts', async () => { + let failUpload: (r: Response) => void = () => {} + const { fetch } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((resolve) => (failUpload = resolve)) + : undefined, + run, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'a.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState()).toMatchObject({ status: 'submitted', conversations: [] }) + failUpload(text('boom', 500)) + await expect(sending).rejects.toThrow('boom') + expect(chat.getState().conversations).toEqual([]) + }) + + test('an already aborted signal uploads nothing', async () => { + const { fetch, calls } = fetchMock(upload) + const api = new WindmillChatApi({ baseUrl: BASE, workspace: 'ws', token: 'tok', fetch }) + const controller = new AbortController() + controller.abort() + await expect( + uploadAttachments(api, [{ name: 'a.pdf', data: pdf }], 'turn', controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(calls).toHaveLength(0) + }) + + test('the pending user message carries its uploaded files before the run returns', async () => { + let releaseRun: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + upload, + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? new Promise((resolve) => (releaseRun = resolve)) + : undefined, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read these', { + attachments: [ + { name: 'photo.webp', data: png }, + { name: 'contract', data: pdf } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + while (runs(calls).length === 0) await new Promise((r) => setTimeout(r, 1)) + const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + const pending = chat.getState().messages.find((m) => m.role === 'user')! + expect(pending.pending).toBe(true) + expect(pending.attachments).toEqual([ + { input: 'files', s3: keys[0], filename: 'photo.png' }, + { input: 'files', s3: keys[1], filename: 'contract.pdf' } + ]) + releaseRun(text('job-1')) + await sending + }) + + test('an explicit mediaType wins over the type a data URL declares', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await chat.sendMessage('read this', { + attachments: [ + { + name: 'contract', + data: `data:application/octet-stream;base64,${btoa('%PDF')}`, + mediaType: 'application/pdf' + } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + const call = uploads(calls)[0] + expect(call.url.searchParams.get('file_key')).toMatch(/\/0\/contract\.pdf$/) + expect(call.url.searchParams.get('content_type')).toBe('application/pdf') + }) + + test('stop() after the uploads land but before the run starts runs nothing', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + // The upload responds at once; Stop lands before the send resumes after it. + while (uploads(calls).length === 0) await Promise.resolve() + await chat.stop() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(calls.filter((c) => c.method === 'DELETE')).toEqual([]) + expect(chat.getState()).toMatchObject({ status: 'idle', messages: [], conversations: [] }) + }) + + test('a subscriber stopping when the attachments appear runs nothing', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + chat.subscribe((s) => { + if (s.messages.some((m) => m.attachments)) void chat.stop() + }) + await expect( + chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(calls.filter((c) => c.method === 'DELETE')).toEqual([]) + expect(chat.getState()).toMatchObject({ status: 'idle', messages: [], conversations: [] }) + }) +}) diff --git a/chat-sdk/test/chat.test.ts b/chat-sdk/test/chat.test.ts new file mode 100644 index 0000000000..a739abf3ad --- /dev/null +++ b/chat-sdk/test/chat.test.ts @@ -0,0 +1,1170 @@ +import { describe, expect, test } from 'bun:test' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { fetchMock, json, memoryStorage, messageRow, ndjson, sse, sseTimed, text, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined + +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +function options(extra: Partial, fetch: ChatOptions['fetch']): ChatOptions { + return { flowPath: FLOW, baseUrl: BASE, workspace: 'ws', fetch, storage: memoryStorage(), ...extra } +} + +describe('createChat with local history', () => { + test('streams text and tool calls, then finalizes and persists the turn', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { type: 'ping' }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' } + ), + stream_offset: 3, + flow_stream_job_id: 'agent-job' + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'token_delta', content: 'The answer is 42' } + ), + stream_offset: 5, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await chat.sendMessage('what is the answer?', { inputs: { locale: 'fr' } }) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.url.searchParams.get('memory_id')).toBe(chat.getState().conversationId!) + expect(runCall.body).toEqual({ locale: 'fr', user_message: 'what is the answer?' }) + expect(runCall.headers.authorization).toBe('Bearer tok') + + const state = chat.getState() + expect(state.status).toBe('idle') + expect(statuses).toContain('submitted') + expect(statuses).toContain('streaming') + expect(state.messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'what is the answer?', false], + ['assistant', 'Let me ', false], + ['tool', 'Used lookup tool', false], + ['assistant', 'The answer is 42', false] + ]) + expect(state.messages[2].tool).toEqual({ + callId: 'c1', + name: 'lookup', + status: 'success', + arguments: '{"q":1}', + result: '42' + }) + expect(state.conversations).toHaveLength(1) + expect(state.conversations[0].title).toBe('what is the answer?') + + const reloaded = createChat(options({ token: 'tok', storage }, fetch)) + await reloaded.loadConversations() + expect(reloaded.getState().conversations.map((c) => c.id)).toEqual([state.conversationId!]) + await reloaded.selectConversation(state.conversationId!) + expect(reloaded.getState().messages.map((m) => m.content)).toEqual( + state.messages.map((m) => m.content) + ) + }) + + test('a tool call id reused by a later turn gets its own message', async () => { + const toolTurn = () => + sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'same-id', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'same-id', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'done' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'done', messages: [] } + } + ]) + const { fetch } = fetchMock(run, (c) => (c.url.pathname === streamPath ? toolTurn() : undefined)) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('one') + await chat.sendMessage('two') + expect(chat.getState().messages.map((m) => m.role)).toEqual([ + 'user', + 'tool', + 'assistant', + 'user', + 'tool', + 'assistant' + ]) + }) + + test('a custom run replaces the deployed flow call and still follows the job', async () => { + const { fetch, calls } = fetchMock((c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'from preview' } }]) + : undefined + ) + const seen: unknown[] = [] + const chat = createChat( + options( + { + token: 'tok', + inputs: { tone: 'kind' }, + run: async (args, turn) => { + seen.push({ args, conversationId: turn.conversationId, aborted: turn.signal.aborted }) + return 'job-1' + } + }, + fetch + ) + ) + await chat.sendMessage('hi') + expect(seen).toEqual([{ args: { tone: 'kind', user_message: 'hi' }, conversationId: chat.getState().conversationId, aborted: false }]) + expect(calls.filter((c) => c.method === 'POST')).toHaveLength(0) + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'from preview']) + }) + + test('re-attaches from the start when the streaming step is retried under a new sub-job', async () => { + let streams = 0 + const { fetch, calls } = fetchMock(run, (c) => { + if (c.url.pathname !== streamPath) return undefined + streams++ + if (streams === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'first try ' }), stream_offset: 1, flow_stream_job_id: 'agent-1' }, + // The retried step streams under a new sub-job; the offset above indexes the old one. + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'y ' }), stream_offset: 2, flow_stream_job_id: 'agent-2' } + ]) + } + return sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'second try' }), + stream_offset: 1, + flow_stream_job_id: 'agent-2', + completed: true, + only_result: 'second try' + } + ]) + }) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const streamCalls = calls.filter((c) => c.url.pathname === streamPath) + expect(streamCalls).toHaveLength(2) + expect(streamCalls[1].url.searchParams.get('stream_offset')).toBeNull() + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'first try second try']) + }) + + test('resumes after a stream timeout from the last offset without re-running the flow', async () => { + let streamCalls = 0 + const { fetch, calls } = fetchMock(run, (c) => { + if (c.url.pathname !== streamPath) return undefined + streamCalls++ + if (streamCalls === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'Hel' }), stream_offset: 1 }, + { type: 'timeout' } + ]) + } + return sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'lo' }), + stream_offset: 2, + completed: true, + only_result: 'Hello' + } + ]) + }) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + + expect(calls.filter((c) => c.method === 'POST')).toHaveLength(1) + const streams = calls.filter((c) => c.url.pathname === streamPath) + expect(streams).toHaveLength(2) + expect(streams[0].url.searchParams.get('stream_offset')).toBeNull() + expect(streams[1].url.searchParams.get('stream_offset')).toBe('1') + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'Hello']) + }) + + test('derives the answer from the flow result when nothing streamed', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const [, answer] = chat.getState().messages + expect(answer.role).toBe('assistant') + expect(answer.content).toBe('From a script') + expect(answer.jobId).toBe('job-1') + }) + + test('renders a successful result that merely looks like an error envelope', async () => { + const result = { error: { message: 'domain data' } } + const { fetch } = fetchMock( + run, + (c) => (c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: result }]) : undefined), + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: true, result }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages[1]).toMatchObject({ + role: 'assistant', + success: true, + content: JSON.stringify(result, null, 2) + }) + }) + + test('reports a failed flow as an unsuccessful assistant message', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + completed: true, + only_result: { error: { name: 'ExecutionErr', message: 'boom' } } + } + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false, result: { error: { message: 'boom' } } }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages[1]).toMatchObject({ role: 'assistant', success: false, content: 'ExecutionErr: boom' }) + }) +}) + +describe('createChat with server history', () => { + test('replaces the optimistic turn with the persisted rows', async () => { + const { fetch, calls } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'hmm' }, + { type: 'token_delta', content: 'Hello' } + ), + stream_offset: 2, + completed: true, + only_result: { output: 'Hello', messages: [] } + } + ]) + : undefined, + (c) => + c.method === 'GET' && c.url.pathname.endsWith('/messages') + ? json([ + messageRow(11, 'user', 'hi'), + messageRow(12, 'assistant', 'Hello', { step_name: 'AI Agent', job_id: 'agent-job' }) + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json([ + { + id: chatId, + workspace_id: 'ws', + flow_path: FLOW, + title: 'hi', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:01Z', + created_by: 'admin' + } + ]) + : undefined + ) + const chat = createChat(options({}, fetch)) + let chatId = '' + const unsubscribe = chat.subscribe((s) => { + chatId = s.conversationId ?? chatId + }) + await chat.sendMessage('hi') + unsubscribe() + + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-11', 'user', 'hi', false], + ['row-12', 'assistant', 'Hello', false] + ]) + // Ids stay the client's, so list keys never remount; the server id rides alongside. + expect(state.messages.map((m) => m.id.startsWith('pending-'))).toEqual([true, true]) + expect(state.messages[1]).toMatchObject({ reasoning: 'hmm', stepName: 'AI Agent', jobId: 'agent-job', seq: 12 }) + expect(state.conversations.map((c) => c.id)).toEqual([chatId]) + + const messagesCall = calls.find((c) => c.url.pathname.endsWith('/messages'))! + expect(messagesCall.url.pathname).toBe(`/api/w/ws/flow_conversations/${chatId}/messages`) + expect(messagesCall.headers.authorization).toBeUndefined() + }) + + test('a persisted row brings back its attachments, reasoning and the call an MCP tool row carries', async () => { + const { fetch } = fetchMock( + (c) => + c.method === 'GET' && c.url.pathname === '/api/w/ws/flow_conversations/conv-1/messages' + ? json([ + messageRow(1, 'user', 'hi', { + attachments: [{ input: 'files', s3: 'chat/a.png', storage: 'secondary', filename: 'a.png' }] + }), + messageRow(2, 'tool', 'Used lookup tool', { + job_id: 'agent-job', + tool_arguments: '{"q":1}', + tool_result: '42' + }), + messageRow(3, 'tool', 'Error executing lookup', { + job_id: 'agent-job', + success: false, + tool_arguments: '{"q":2}', + tool_result: 'MCP tool error: boom' + }), + messageRow(4, 'assistant', 'The answer is 42', { reasoning: 'hmm' }), + messageRow(5, 'assistant', 'Hello'), + messageRow(6, 'tool', 'Used get_price tool', { + job_id: 'script-tool-job', + tool_arguments: '{"item":"widget"}', + tool_result: '{"price":42}' + }) + ]) + : undefined + ) + const chat = createChat(options({ history: 'server' }, fetch)) + await chat.selectConversation('conv-1') + + const [user, used, failed, answer, plain, scriptTool] = chat.getState().messages + expect(scriptTool).toMatchObject({ jobId: 'script-tool-job', tool: { name: 'get_price', status: 'success', arguments: '{"item":"widget"}', result: '{"price":42}' } }) + expect(user.attachments).toEqual([{ input: 'files', s3: 'chat/a.png', storage: 'secondary', filename: 'a.png' }]) + expect(answer.attachments).toBeUndefined() + expect(used.tool).toEqual({ name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' }) + expect(failed.tool).toEqual({ name: 'lookup', status: 'error', arguments: '{"q":2}', result: 'MCP tool error: boom' }) + expect(answer.reasoning).toBe('hmm') + expect(plain.reasoning).toBeUndefined() + }) + + test('a row nothing streamed, like a web search, lands before the answer as on reload', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'Rust.' }), + stream_offset: 1, + completed: true, + only_result: { output: 'Rust.', messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([ + messageRow(91, 'user', 'hi'), + messageRow(92, 'tool', 'Used websearch tool', { job_id: 'step-1', tool_result: '[{"url":"https://example.com"}]' }), + messageRow(93, 'assistant', 'Rust.', { job_id: 'step-1' }) + ]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.seq])).toEqual([ + ['user', 'hi', 91], + ['tool', 'Used websearch tool', 92], + ['assistant', 'Rust.', 93] + ]) + }) + + test('a failure row stays after streamed text that never got a row', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'Let me look' }), + stream_offset: 1, + completed: true, + only_result: { error: { name: 'ExecutionErr', message: 'boom' } } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(91, 'user', 'hi'), messageRow(92, 'assistant', 'boom', { job_id: 'step-1', success: false })]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + const messages = chat.getState().messages + expect(messages.map((m) => [m.content, m.success])).toEqual([ + ['hi', true], + ['Let me look', true], + ['boom', false] + ]) + }) + + test('keeps the streamed answer until its row lands, even when a tool row lands first', async () => { + let messageFetches = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'Final answer' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'Final answer', messages: [] } + } + ]) + : undefined, + (c) => { + if (!c.url.pathname.endsWith('/messages')) return undefined + messageFetches++ + // The assistant row is written by a task that trails the tool's. + return json( + messageFetches === 1 + ? [messageRow(21, 'user', 'hi'), messageRow(22, 'tool', 'Used lookup tool')] + : [messageRow(23, 'assistant', 'Final answer')] + ) + }, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(messageFetches).toBe(2) + expect(chat.getState().messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-21', 'user', 'hi', false], + ['row-22', 'tool', 'Used lookup tool', false], + ['row-23', 'assistant', 'Final answer', false] + ]) + expect(chat.getState().messages[1].tool).toMatchObject({ callId: 'c1', result: '1', status: 'success' }) + }) + + test('finishes the turn from the flow result when history falls back mid-turn', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('forbidden', 403) : undefined) + ) + const chat = createChat(options({ storage }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('local') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + const stored = JSON.parse([...storage.data.values()][0]) + expect(stored.messages[state.conversationId!]).toHaveLength(2) + }) + + test('appends later pages of conversations', async () => { + const row = (id: string) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin' + }) + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json(c.url.searchParams.get('page') === '2' ? [row('c2')] : [row('c1')]) + : undefined + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations() + await chat.loadConversations({ page: 2 }) + expect(chat.getState().conversations.map((c) => c.id)).toEqual(['c1', 'c2']) + }) + + test('lists one kind of conversation and carries which kind each one is', async () => { + const row = (id: string, is_test: boolean) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin', + is_test + }) + const { fetch, calls } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json(c.url.searchParams.get('kind') === 'test' ? [row('t1', true)] : [row('d1', false)]) + : undefined + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations() + expect(calls[0].url.searchParams.has('kind')).toBe(false) + expect(chat.getState().conversations.map((c) => [c.id, c.isTest])).toEqual([['d1', false]]) + await chat.loadConversations({ kind: 'test' }) + expect(calls[1].url.searchParams.get('kind')).toBe('test') + expect(chat.getState().conversations.map((c) => [c.id, c.isTest])).toEqual([['t1', true]]) + }) + + test('a list for a kind no longer asked for does not replace the newer one', async () => { + const row = (id: string, is_test: boolean) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin', + is_test + }) + const { fetch } = fetchMock((c) => { + if (c.url.pathname !== '/api/w/ws/flow_conversations/list') return undefined + if (c.url.searchParams.get('kind') === 'test') { + return new Promise((r) => setTimeout(() => r(json([row('t1', true)])), 50)) + } + return json([row('d1', false)]) + }) + const chat = createChat(options({}, fetch)) + const slow = chat.loadConversations({ kind: 'test' }) + await chat.loadConversations({ kind: 'deployed' }) + await slow + expect(chat.getState().conversations.map((c) => c.id)).toEqual(['d1']) + // Another kind asked for on a later page starts its own listing rather than appending. + await chat.loadConversations({ page: 2, kind: 'test' }) + expect(chat.getState().conversations.map((c) => c.id)).toEqual(['t1']) + }) + + test('the refresh after a new turn lists the kind last asked for', async () => { + const { fetch, calls } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Hello', messages: [] } }]) + : undefined, + (c) => + c.method === 'GET' && c.url.pathname.endsWith('/messages') + ? json([messageRow(11, 'user', 'hi'), messageRow(12, 'assistant', 'Hello', { job_id: 'agent-job' })]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations({ kind: 'test' }) + await chat.sendMessage('hi') + const lists = calls.filter((c) => c.url.pathname === '/api/w/ws/flow_conversations/list') + expect(lists.length).toBeGreaterThan(1) + expect(lists.every((c) => c.url.searchParams.get('kind') === 'test')).toBe(true) + }) + + test('renaming a conversation keeps its place in the list', async () => { + const row = (id: string) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin', + is_test: false + }) + const { fetch, calls } = fetchMock( + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([row('c1'), row('c2')]) : undefined), + (c) => + c.method === 'POST' && c.url.pathname === '/api/w/ws/flow_conversations/update/c2' + ? text('Conversation c2 updated') + : undefined + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations() + await chat.renameConversation('c2', ' Budget review ') + expect(calls[1].body).toEqual({ title: 'Budget review' }) + expect(chat.getState().conversations.map((c) => [c.id, c.title])).toEqual([ + ['c1', 'c1'], + ['c2', 'Budget review'] + ]) + // Cut as the server cuts, so what is shown is what is stored. + await chat.renameConversation('c2', 'x'.repeat(300)) + expect(chat.getState().conversations[1].title).toBe('x'.repeat(252) + '...') + expect(calls[2].body).toEqual({ title: 'x'.repeat(252) + '...' }) + }) + + test('a turn started right after stop() is not touched by the stop sync', async () => { + let jobs = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'slow...' }), stream_offset: 1 }]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c2', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c2', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'second' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'second', messages: [] } + } + ]) + : undefined, + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('ok') : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(31, 'user', 'first'), messageRow(32, 'user', 'second question'), messageRow(33, 'tool', 'Used lookup tool'), messageRow(34, 'assistant', 'second')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + // The stream of job-1 never completes: the connection just ends, so the turn keeps waiting. + await new Promise((r) => setTimeout(r, 50)) + const stopped = chat.stop() + await first + const second = chat.sendMessage('second question') + await stopped + await second + const roles = chat.getState().messages.map((m) => `${m.role}${m.pending ? '*' : ''}`) + expect(roles).toEqual(['user', 'assistant', 'user', 'tool', 'assistant']) + expect(chat.getState().messages.filter((m) => m.role === 'tool')).toHaveLength(1) + }) + + test('answers from the flow result when only the user row has been persisted', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/messages') ? (reads++, json([messageRow(41, 'user', 'hi')])) : undefined), + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBeGreaterThan(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-41'], + ['assistant', 'From a script', undefined] + ]) + }) + + test('an answer the poller merged before completion is not appended again', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sseTimed([{ type: 'update' }, 1400, { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(51, 'user', 'hi'), messageRow(52, 'assistant', 'From a script')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-51'], + ['assistant', 'From a script', 'row-52'] + ]) + }) + + test('an earlier round of a non-streaming agent is not its answer', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Final answer', messages: [] } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-1') + ? json({ flow_status: { modules: [{ job: 'step-1', agent_actions: [{ type: 'tool_call', job_id: 'tool-1' }, { type: 'message' }] }] } }) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(91, 'user', 'hi'), messageRow(92, 'assistant', 'Let me check', { job_id: 'step-1' }), messageRow(93, 'tool', 'Used lookup tool', { job_id: 'tool-1' })] + : [messageRow(94, 'assistant', 'Final answer', { job_id: 'step-1' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-91'], + ['assistant', 'Let me check', 'row-92'], + ['tool', 'Used lookup tool', 'row-93'], + ['assistant', 'Final answer', 'row-94'] + ]) + }) + + test('an answer persisted in another shape than the flow result is still the answer', async () => { + // An image agent returns the S3 object and persists it with a type marker. + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { s3: 'agent/img.png' } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/jobs_u/get/job-1') ? json({ flow_status: { modules: [{ job: 'step-1' }] } }) : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json(++reads === 1 ? [messageRow(71, 'user', 'draw'), messageRow(72, 'assistant', '{"s3":"agent/img.png","type":"windmill_s3_object"}', { job_id: 'step-1' })] : []) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('draw') + expect(reads).toBe(1) + expect(chat.getState().messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'draw'], + ['assistant', '{"s3":"agent/img.png","type":"windmill_s3_object"}'] + ]) + }) + + test('a tool row without a job (an MCP call) still separates a round from the answer', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Final answer', messages: [] } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/jobs_u/get/job-1') ? json({ flow_status: { modules: [{ job: 'step-1', agent_actions: [{ type: 'mcp_tool_call' }, { type: 'message' }] }] } }) : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(81, 'user', 'hi'), messageRow(82, 'assistant', 'Let me check', { job_id: 'step-1' }), messageRow(83, 'tool', 'Used search tool', { job_id: null })] + : [messageRow(84, 'assistant', 'Final answer', { job_id: 'step-1' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'Let me check', 'Used search tool', 'Final answer']) + }) + + test('thinking that led to a tool call rides on the call, live and once its row lands', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + // No `tool_call_arguments`: a stream cut short leaves the call without them. + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'r1' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'Final' } + ), + stream_offset: 4, + completed: true, + only_result: { output: 'Final', messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([ + messageRow(71, 'user', 'hi'), + messageRow(72, 'tool', 'Used lookup tool', { job_id: 'step-1', reasoning: 'r1', tool_arguments: '{"q":1}', tool_result: '1' }), + messageRow(73, 'assistant', 'Final', { job_id: 'step-1' }) + ]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + const messages = chat.getState().messages + expect(messages.map((m) => [m.role, m.content, m.reasoning, m.seq])).toEqual([ + ['user', 'hi', undefined, 71], + ['tool', 'Used lookup tool', 'r1', 72], + ['assistant', 'Final', undefined, 73] + ]) + expect(messages[1].tool).toMatchObject({ callId: 'c1', arguments: '{"q":1}', result: '1', status: 'success' }) + }) + + test('a structured answer row replaces the call it streamed as', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'hmm' }, + { type: 'tool_call', call_id: 'c9', function_name: 'structured_output' }, + { type: 'tool_call_arguments', call_id: 'c9', function_name: 'structured_output', arguments: '{"n": 1}' }, + { type: 'tool_execution', call_id: 'c9', function_name: 'structured_output' } + ), + stream_offset: 4, + completed: true, + only_result: { output: { n: 1 }, messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(75, 'user', 'hi'), messageRow(76, 'assistant', '{"n": 1}', { job_id: 'step-1', reasoning: 'hmm' })]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.reasoning, m.tool])).toEqual([ + ['user', 'hi', undefined, undefined], + ['assistant', '{"n": 1}', 'hmm', undefined] + ]) + }) + + test('a structured answer row leaves a stopped turn its identical call', async () => { + const call = (id: string) => + ndjson( + { type: 'tool_call', call_id: id, function_name: 'structured_output' }, + { type: 'tool_call_arguments', call_id: id, function_name: 'structured_output', arguments: '{"ok": true}' } + ) + let jobs = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') ? sse([{ type: 'update', new_result_stream: call('c1'), stream_offset: 2 }]) : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', new_result_stream: call('c2'), stream_offset: 2, completed: true, only_result: { output: { ok: true }, messages: [] } }]) + : undefined, + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('ok') : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(41, 'user', 'first'), messageRow(42, 'user', 'again'), messageRow(43, 'assistant', '{"ok": true}', { job_id: 'step-2' })]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 50)) + const stopped = chat.stop() + await first + const second = chat.sendMessage('again') + await stopped + await second + expect(chat.getState().messages.map((m) => [m.role, m.content, m.tool?.callId])).toEqual([ + ['user', 'first', undefined], + ['tool', '', 'c1'], + ['user', 'again', undefined], + ['assistant', '{"ok": true}', undefined] + ]) + }) + + test('the stream asks for a server poll interval only when one is set', async () => { + const answer: Route = (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + const plain = fetchMock(run, answer) + await createChat(options({ token: 'tok' }, plain.fetch)).sendMessage('hi') + expect(plain.calls.find((c) => c.url.pathname === streamPath)!.url.searchParams.has('poll_delay_ms')).toBe(false) + const fast = fetchMock(run, answer) + await createChat(options({ token: 'tok', pollDelayMs: 50 }, fast.fetch)).sendMessage('hi') + expect(fast.calls.find((c) => c.url.pathname === streamPath)!.url.searchParams.get('poll_delay_ms')).toBe('50') + }) + + test('a tool row alone is not the answer of a turn that streamed no text', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Answer', messages: [] } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json(++reads === 1 ? [messageRow(61, 'user', 'hi'), messageRow(62, 'tool', 'Used lookup tool')] : [messageRow(63, 'assistant', 'Answer')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-61'], + ['tool', 'Used lookup tool', 'row-62'], + ['assistant', 'Answer', 'row-63'] + ]) + }) + + test('a late answer from a stopped job is not taken as the next turn answer', async () => { + let jobs = 0 + let reads = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + // job-1 never completes: the connection just ends, so the turn keeps waiting. + (c) => (c.url.pathname.endsWith('/getupdate_sse/job-1') ? sse([{ type: 'update' }]) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'second answer' } }]) + : undefined, + // The run-only token cannot cancel: job-1 keeps running after stop(). + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('forbidden', 400) : undefined), + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-2') + ? json({ flow_status: { modules: [{ job: 'step-2' }] } }) + : undefined, + // Read 1 is stop()'s sync; the stopped job's answer lands after the second user row. + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(71, 'user', 'first')] + : reads === 2 + ? [messageRow(72, 'user', 'second'), messageRow(73, 'assistant', 'first answer, late', { job_id: 'step-1' })] + : [messageRow(74, 'assistant', 'second answer', { job_id: 'step-2' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 50)) + await chat.stop() + await first + await chat.sendMessage('second') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'first', 'row-71'], + ['user', 'second', 'row-72'], + ['assistant', 'first answer, late', 'row-73'], + ['assistant', 'second answer', 'row-74'] + ]) + }) + + test('a failure handler answer is attributed to the turn', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-1') + ? json({ flow_status: { modules: [{ job: 'step-1' }], failure_module: { job: 'handler-1' } } }) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? (reads++, json([messageRow(81, 'user', 'hi'), messageRow(82, 'assistant', 'Sorry: boom', { job_id: 'handler-1', success: false })])) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.success, m.serverId])).toEqual([ + ['user', 'hi', true, 'row-81'], + ['assistant', 'Sorry: boom', false, 'row-82'] + ]) + }) + + test('deleting the current local conversation mid-turn leaves nothing behind', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 300)) + const id = chat.getState().conversationId! + await chat.deleteConversation(id) + await turn + await new Promise((r) => setTimeout(r, 400)) + expect(chat.getState().conversations).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages).toEqual([]) + expect([...storage.data.values()].join('')).not.toContain('hello') + }) + + test('viewing an older local conversation does not reorder history', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + await chat.sendMessage('older') + const older = chat.getState().conversationId! + chat.newConversation() + await chat.sendMessage('newer') + const newer = chat.getState().conversationId! + await chat.selectConversation(older) + await new Promise((r) => setTimeout(r, 400)) + const again = createChat(options({ token: 'tok', storage }, fetch)) + expect((await again.loadConversations()).map((c) => c.id)).toEqual([newer, older]) + }) + + test('renaming a local conversation persists the title without reordering history', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + await chat.sendMessage('older') + const older = chat.getState().conversationId! + chat.newConversation() + await chat.sendMessage('newer') + const newer = chat.getState().conversationId! + const before = calls.length + await chat.renameConversation(older, 'Renamed') + expect(calls.length).toBe(before) + const again = createChat(options({ token: 'tok', storage }, fetch)) + expect((await again.loadConversations()).map((c) => [c.id, c.title])).toEqual([ + [newer, 'newer'], + [older, 'Renamed'] + ]) + }) + + test('destroying the chat mid-turn leaves it idle', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + expect(chat.getState().status).toBe('streaming') + chat.destroy() + await turn + expect(chat.getState().status).toBe('idle') + expect(chat.getState().messages.every((m) => !m.pending)).toBe(true) + }) + + test('destroying the chat during a local turn keeps what it showed', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.destroy() + await turn + const again = createChat(options({ token: 'tok', storage }, fetch)) + await again.loadConversations() + expect(again.getState().conversations.map((c) => c.id)).toEqual([id]) + await again.selectConversation(id) + expect(again.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('switching conversations keeps what a local turn showed so far', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.newConversation() + await turn + expect(chat.getState().messages).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('answers from the flow result when server history keeps failing', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('down', 503) : undefined) + ) + const chat = createChat(options({ history: 'server' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + }) + + test('falls back to local history when the credential cannot read conversations', async () => { + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' ? text('forbidden', 403) : undefined + ) + const chat = createChat(options({}, fetch)) + expect(chat.getState().history).toBe('server') + await chat.loadConversations() + expect(chat.getState().history).toBe('local') + + const explicit = createChat(options({ history: 'server' }, fetch)) + await expect(explicit.loadConversations()).rejects.toThrow('403') + expect(explicit.getState().history).toBe('server') + }) +}) diff --git a/chat-sdk/test/config.test.ts b/chat-sdk/test/config.test.ts new file mode 100644 index 0000000000..2289f20c89 --- /dev/null +++ b/chat-sdk/test/config.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { resolveConfig } from '../src/config' + +const g = globalThis as { process?: unknown; ctx?: unknown; location?: unknown } +const originalProcess = g.process + +afterEach(() => { + g.process = originalProcess + delete g.ctx + delete g.location +}) + +describe('resolveConfig', () => { + test('explicit token defaults history to local; a session defaults to server', () => { + const base = { flowPath: 'f/a/b', baseUrl: 'http://wm.test/', workspace: 'ws' } + expect(resolveConfig({ ...base, token: 't' }).history).toBe('local') + expect(resolveConfig(base).history).toBe('server') + expect(resolveConfig({ ...base, token: 't', history: 'server' }).historyExplicit).toBe(true) + }) + + test('reads the sandboxed raw app env the wrapper injects', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: 'sdk-token', history: 'server' }) + }) + + test('keeps the raw app token off another instance', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://other.test', workspace: 'ws' }).token).toBeUndefined() + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://wm.test' }).token).toBe('sdk-token') + }) + + test('reads the unsandboxed raw app context and uses the page origin', () => { + g.ctx = { ctx: { username: 'admin' }, workspace: 'ws' } + g.location = { origin: 'http://wm.test' } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: undefined }) + }) + + test('refuses an opaque origin without an SDK token', () => { + g.ctx = { workspace: 'ws' } + g.location = { origin: 'null' } + expect(() => resolveConfig({ flowPath: 'f/a/b' })).toThrow('frontend SDK scopes') + }) +}) diff --git a/chat-sdk/test/follow.test.ts b/chat-sdk/test/follow.test.ts new file mode 100644 index 0000000000..19af6baf38 --- /dev/null +++ b/chat-sdk/test/follow.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from 'bun:test' +import { WindmillChatApi } from '../src/api' +import { followJob } from '../src/follow' +import { fetchMock, ndjson, sse } from './support' + +test('a retried streaming step reports its offset as lost before the new sub-job is followed', async () => { + let streams = 0 + const { fetch } = fetchMock((c) => { + if (!c.url.pathname.endsWith('/jobs_u/getupdate_sse/job-1')) return undefined + streams++ + if (streams === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'a' }), stream_offset: 3, flow_stream_job_id: 'agent-1' }, + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'b' }), stream_offset: 4, flow_stream_job_id: 'agent-2' } + ]) + } + return sse([{ type: 'update', stream_offset: 1, flow_stream_job_id: 'agent-2', completed: true, only_result: 'ok' }]) + }) + const api = new WindmillChatApi({ baseUrl: 'http://wm.test', workspace: 'ws', token: 'tok', fetch }) + const offsets: (number | undefined)[] = [] + for await (const _ of followJob(api, 'job-1', { onOffset: (o) => offsets.push(o) })) { + // A resumer that stored offset 3 must not reuse it against agent-2. + } + expect(offsets).toEqual([3, undefined, 1]) +}) diff --git a/chat-sdk/test/react.test.tsx b/chat-sdk/test/react.test.tsx new file mode 100644 index 0000000000..75773ad0f2 --- /dev/null +++ b/chat-sdk/test/react.test.tsx @@ -0,0 +1,103 @@ +import { GlobalRegistrator } from '@happy-dom/global-registrator' +// Test files share one process: the DOM globals must not outlive this file. +GlobalRegistrator.register() + +import { afterAll, describe, expect, test } from 'bun:test' +import React, { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { useWindmillChat, type UseWindmillChat } from '../src/react' +import type { ChatOptions } from '../src/types' +import { fetchMock, memoryStorage } from './support' + +afterAll(() => GlobalRegistrator.unregister()) + +const base: ChatOptions = { + flowPath: 'f/chat/agent', + baseUrl: 'http://wm.test', + workspace: 'ws', + fetch: fetchMock().fetch, + storage: memoryStorage() +} + +/** Renders the hook and hands back what it returned, rerendering with new options on demand. */ +function mountHook() { + let latest: UseWindmillChat | undefined + const Probe = (props: ChatOptions) => { + latest = useWindmillChat(props) + return null + } + const root: Root = createRoot(document.createElement('div')) + const render = (props: ChatOptions) => { + act(() => root.render()) + return latest! + } + return { render, unmount: () => act(() => root.unmount()) } +} + +describe('useWindmillChat', () => { + test('a credential change is a new chat; a new closure for the same credential is not', () => { + const { render, unmount } = mountHook() + const a = render({ ...base, token: 'user-a' }).chat + expect(render({ ...base, token: 'user-a' }).chat).toBe(a) + const b = render({ ...base, token: 'user-b' }).chat + expect(b).not.toBe(a) + + const fn1 = render({ ...base, token: () => 'fn-1' }).chat + expect(fn1).not.toBe(b) + expect(render({ ...base, token: () => 'fn-2' }).chat).toBe(fn1) + + const session = render({ ...base }).chat + expect(session).not.toBe(fn1) + const fn3 = render({ ...base, token: () => 'fn-3' }).chat + expect(fn3).not.toBe(session) + expect(render({ ...base, token: () => 'fn-3', storageKey: 'someone-else' }).chat).not.toBe(fn3) + unmount() + }) + + test('the latest inputs go with the next message', async () => { + const { fetch, calls } = fetchMock((c) => (c.method === 'POST' ? new Response('job-1') : undefined)) + const { render, unmount } = mountHook() + render({ ...base, fetch, token: 'tok', inputs: { docId: 'first' } }) + const hook = render({ ...base, fetch, token: 'tok', inputs: { docId: 'second' } }) + // The run's stream never answers here; only the request matters. + void hook.sendMessage('hi', { inputs: { extra: true } }).catch(() => {}) + await new Promise((r) => setTimeout(r, 20)) + expect(calls.find((c) => c.method === 'POST')?.body).toEqual({ docId: 'second', extra: true, user_message: 'hi' }) + hook.chat.destroy() + // A key the latest render no longer passes is gone from the next message. + const cleared = render({ ...base, fetch, token: 'tok', inputs: {} }) + void cleared.sendMessage('again').catch(() => {}) + await new Promise((r) => setTimeout(r, 20)) + expect(calls.filter((c) => c.method === 'POST')[1]?.body).toEqual({ user_message: 'again' }) + unmount() + }) + + test('the latest render’s run callback starts the next turn', async () => { + const { render, unmount } = mountHook() + const started: string[] = [] + const runner = (name: string) => async () => { + started.push(name) + throw new Error('stop here') + } + const first = render({ ...base, token: 'tok', run: runner('first') }) + const second = render({ ...base, token: 'tok', run: runner('second') }) + expect(second.chat).toBe(first.chat) + await act(() => second.sendMessage('hi').catch(() => {})) + expect(started).toEqual(['second']) + // Dropping the runner means the deployed flow again: a different chat. + expect(render({ ...base, token: 'tok' }).chat).not.toBe(first.chat) + unmount() + }) + + test('a token function is read through a ref, so the latest closure serves the next request', async () => { + const { fetch, calls } = fetchMock((c) => (c.url.pathname.includes('/flow_conversations/list') ? new Response('[]') : undefined)) + const { render, unmount } = mountHook() + const first = render({ ...base, fetch, history: 'server', token: () => 'first' }) + await act(() => first.loadConversations()) + const second = render({ ...base, fetch, history: 'server', token: () => 'second' }) + expect(second.chat).toBe(first.chat) + await act(() => second.loadConversations()) + expect(calls.map((c) => c.headers.authorization)).toEqual(['Bearer first', 'Bearer second']) + unmount() + }) +}) diff --git a/chat-sdk/test/stream.test.ts b/chat-sdk/test/stream.test.ts new file mode 100644 index 0000000000..84f2502133 --- /dev/null +++ b/chat-sdk/test/stream.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from 'bun:test' +import { readServerSentEvents } from '../src/api' +import { createStreamEventParser, parseStreamEvents } from '../src/stream' +import { ndjson } from './support' + +describe('parseStreamEvents', () => { + test('keeps agent events and skips other lines', () => { + const events = parseStreamEvents( + ndjson( + { type: 'token_delta', content: 'Hi' }, + { type: 'reasoning_token_delta', content: 'thinking' }, + { type: 'something_else', content: 'x' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true } + ) + 'not json\n' + ) + expect(events.map((e) => e.type)).toEqual(['token_delta', 'reasoning_token_delta', 'tool_result']) + }) +}) + +describe('createStreamEventParser', () => { + test('holds an incomplete line until the rest arrives', () => { + const parser = createStreamEventParser() + const line = JSON.stringify({ type: 'token_delta', content: 'Hello' }) + expect(parser.push(line.slice(0, 10))).toEqual([]) + expect(parser.push(line.slice(10) + '\n' + '{"type":"token_delta",')).toEqual([ + { type: 'token_delta', content: 'Hello' } + ]) + expect(parser.push('"content":"!"}')).toEqual([]) + expect(parser.flush()).toEqual([{ type: 'token_delta', content: '!' }]) + }) +}) + +describe('readServerSentEvents', () => { + test('splits frames that straddle chunks and normalizes CRLF', async () => { + const chunks = ['data: {"a":1}\r\n\r\ndata: {"b"', ':2}\n\ndata: first\ndata: second\n\n', 'data: {"c":3}'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['{"a":1}', '{"b":2}', 'first\nsecond', '{"c":3}']) + }) + + test('keeps a CRLF split across chunks from ending the event', async () => { + const chunks = ['data: first\r', '\ndata: second\r\n\r\ndata: last\r'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['first\nsecond', 'last']) + }) +}) diff --git a/chat-sdk/test/support.ts b/chat-sdk/test/support.ts new file mode 100644 index 0000000000..e198f19355 --- /dev/null +++ b/chat-sdk/test/support.ts @@ -0,0 +1,106 @@ +import type { FetchLike, StorageLike } from '../src/types' + +export interface RecordedCall { + method: string + url: URL + headers: Record + body: unknown + /** A body sent as is rather than as JSON (an upload). */ + raw?: Blob + signal?: AbortSignal +} + +export type Route = (call: RecordedCall) => Response | Promise | undefined + +/** A fetch whose responses come from the first route that answers; every call is recorded. */ +export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: RecordedCall[] } { + const calls: RecordedCall[] = [] + const fetch: FetchLike = async (input, init) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url) + const call: RecordedCall = { + method: init?.method ?? 'GET', + url, + headers: Object.fromEntries( + Object.entries((init?.headers as Record) ?? {}).map(([k, v]) => [k.toLowerCase(), v]) + ), + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined, + raw: init?.body instanceof Blob ? init.body : undefined, + signal: init?.signal ?? undefined + } + calls.push(call) + for (const route of routes) { + const res = await route(call) + if (res) return res + } + return new Response(`no route for ${call.method} ${url.pathname}`, { status: 404 }) + } + return { fetch, calls } +} + +export function json(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +export function text(value: string, status = 200): Response { + return new Response(value, { status }) +} + +/** A `text/event-stream` body carrying one `data:` frame per event. */ +export function sse(events: object[]): Response { + return new Response(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(''), { + status: 200, + headers: { 'content-type': 'text/event-stream' } + }) +} + +/** Like `sse`, but a number in the list pauses that many milliseconds before the next frame. */ +export function sseTimed(events: (object | number)[]): Response { + const encoder = new TextEncoder() + const body = new ReadableStream({ + async start(controller) { + for (const e of events) { + if (typeof e === 'number') await new Promise((r) => setTimeout(r, e)) + else controller.enqueue(encoder.encode(`data: ${JSON.stringify(e)}\n\n`)) + } + controller.close() + } + }) + return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }) +} + +export function ndjson(...events: object[]): string { + return events.map((e) => JSON.stringify(e)).join('\n') + '\n' +} + +export function memoryStorage(): StorageLike & { data: Map } { + const data = new Map() + return { + data, + getItem: (k) => data.get(k) ?? null, + setItem: (k, v) => void data.set(k, v), + removeItem: (k) => void data.delete(k) + } +} + +export function messageRow( + seq: number, + type: 'user' | 'assistant' | 'tool', + content: string, + extra: Record = {} +) { + return { + id: `row-${seq}`, + conversation_id: 'conv', + message_type: type, + content, + job_id: null, + created_at: '2026-01-01T00:00:00Z', + created_seq: seq, + step_name: null, + success: true, + ...extra + } +} diff --git a/chat-sdk/tsconfig.build.json b/chat-sdk/tsconfig.build.json new file mode 100644 index 0000000000..b8fc4fa324 --- /dev/null +++ b/chat-sdk/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "compilerOptions": { + "types": [], + "noEmit": false, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "rootDir": "src" + } +} diff --git a/chat-sdk/tsconfig.json b/chat-sdk/tsconfig.json new file mode 100644 index 0000000000..fbfee1f0d5 --- /dev/null +++ b/chat-sdk/tsconfig.json @@ -0,0 +1,15 @@ +{ + "include": ["src/**/*", "test/**/*"], + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "types": ["bun"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/chat-sdk/tsdown.config.ts b/chat-sdk/tsdown.config.ts new file mode 100644 index 0000000000..98b23c536e --- /dev/null +++ b/chat-sdk/tsdown.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts', 'src/react.ts', 'src/ai-sdk.ts', 'src/assistant-ui.ts'], + format: ['esm', 'cjs'], + dts: false, + external: ['react', 'ai', '@assistant-ui/react'], + target: 'es2020', +}) diff --git a/cli/TESTING.md b/cli/TESTING.md index 235a95c4b1..df0311677d 100644 --- a/cli/TESTING.md +++ b/cli/TESTING.md @@ -22,6 +22,12 @@ Pure local tests — no backend, no database. Uses `bunfig.unit.toml` (no preloa Examples: `git_unit`, `lint_command_unit`, `tar_creation_unit`, `workspace_conflicts_unit` +`mock.module()` mocks the module for the whole `bun test` process, not for the file +that installs it, and `mock.restore()` does not undo it. A file that mocks a module +must hand back its real exports in `afterAll` (see +`schedule_push_permissioned_as_unit`), or it silently rewires whichever file runs +next — and the run order is the directory's, so it differs between Linux and Windows. + ### Integration tests Require a running backend and PostgreSQL. The `setup.ts` preload builds the backend @@ -29,6 +35,28 @@ binary and starts a shared backend instance. Examples: `sync_pull_push`, `dev_server`, `standalone_commands` +## Module mocks + +`mock.module` replaces a module for the **whole process**, and it does reach modules that +were already imported — a stub one file installs lands on a consumer an earlier file +loaded. + +Handing the module back in `afterAll` is not a reliable undo. Files do run one at a time +(a root-level `afterAll` completes before the next file's body evaluates), so it looks +like it should be — but stubbing `bundle.ts` and restoring it that way still left +`raw_app_svelte_plugin_unit.test.ts` asserting against an empty bundle, green on Linux +and red on Windows, where the `readdir` file order differs. Treat a stub as permanent for +the run. + +So the rule is about what you stub, not how you clean up: **stub only a module no other +in-process suite imports.** Check with `grep -rl "" test/` before reaching +for one. A suite that drives the CLI through a spawned process is out of reach of a +module mock and doesn't count. + +`raw_app_push_policy_unit.test.ts` is the worked example: it stubs `gen/services.gen.ts`, +which passes the rule because nothing else in `test/` imports the three API functions it +replaces, and deliberately does not stub `bundle.ts`, which failed it. + ## AI Benchmark Caveats The repo-level benchmark CLI lives under `ai_evals/`, but it currently depends on diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 1dbc9261de..776726c3a0 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -72,7 +72,7 @@ const packageJson = { url: "https://github.com/windmill-labs/windmill/issues", }, dependencies: { - esbuild: "0.28.0", + esbuild: "0.28.2", ...Object.fromEntries(parserPackages.map(p => [p, cliDeps[p] ?? "*"])), }, optionalDependencies: { diff --git a/cli/bun.lock b/cli/bun.lock index cf3a87bf41..0e0885f563 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -11,7 +11,7 @@ "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", "@windmill-labs/shared-utils": "^1.0.13", "diff": "^5.2.0", - "esbuild": "0.28.0", + "esbuild": "0.28.2", "get-port": "7.1.0", "jszip": "3.8.0", "minimatch": "^10.0.0", @@ -34,7 +34,7 @@ "windmill-parser-wasm-ts": "1.695.0", "windmill-parser-wasm-yaml": "1.770.0", "windmill-yaml-validator": "1.1.1", - "ws": "8.18.0", + "ws": "8.21.3", "yaml": "^2.7.0", }, "devDependencies": { @@ -57,57 +57,57 @@ "@cliffy/table": ["@jsr/cliffy__table@1.0.0", "https://npm.jsr.io/~/11/@jsr/cliffy__table/1.0.0.tgz", { "dependencies": { "@jsr/std__fmt": "^1.0.9" } }, "sha512-VoLxH0DjofHWPWKUc5N+oCwXB6O6e+carnhp23yJTa7qokBb+SCrTIABEgQdIe/p0bxgmZhz17xt2efaAxXvbQ=="], - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ=="], - "@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], + "@esbuild/android-arm": ["@esbuild/android-arm@0.28.2", "", { "os": "android", "cpu": "arm" }, "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg=="], - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="], + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.2", "", { "os": "android", "cpu": "arm64" }, "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A=="], - "@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="], + "@esbuild/android-x64": ["@esbuild/android-x64@0.28.2", "", { "os": "android", "cpu": "x64" }, "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q=="], - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="], + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw=="], - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="], + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw=="], - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="], + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw=="], - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="], + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg=="], - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="], + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.2", "", { "os": "linux", "cpu": "arm" }, "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w=="], - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="], + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug=="], - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="], + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.2", "", { "os": "linux", "cpu": "ia32" }, "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ=="], - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="], + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.2", "", { "os": "linux", "cpu": "none" }, "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ=="], - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="], + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.2", "", { "os": "linux", "cpu": "none" }, "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA=="], - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="], + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ=="], - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="], + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.2", "", { "os": "linux", "cpu": "none" }, "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA=="], - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="], + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg=="], - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="], + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.2", "", { "os": "linux", "cpu": "x64" }, "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ=="], - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="], + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.2", "", { "os": "none", "cpu": "arm64" }, "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw=="], - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="], + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.2", "", { "os": "none", "cpu": "x64" }, "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw=="], - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="], + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ=="], - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="], + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw=="], - "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="], + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.2", "", { "os": "none", "cpu": "arm64" }, "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q=="], - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="], + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g=="], - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="], + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ=="], - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="], + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA=="], - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="], + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.2", "", { "os": "win32", "cpu": "x64" }, "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g=="], "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], @@ -211,7 +211,7 @@ "diff": ["diff@5.2.2", "", {}, "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A=="], - "esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": "bin/esbuild" }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="], + "esbuild": ["esbuild@0.28.2", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.2", "@esbuild/android-arm": "0.28.2", "@esbuild/android-arm64": "0.28.2", "@esbuild/android-x64": "0.28.2", "@esbuild/darwin-arm64": "0.28.2", "@esbuild/darwin-x64": "0.28.2", "@esbuild/freebsd-arm64": "0.28.2", "@esbuild/freebsd-x64": "0.28.2", "@esbuild/linux-arm": "0.28.2", "@esbuild/linux-arm64": "0.28.2", "@esbuild/linux-ia32": "0.28.2", "@esbuild/linux-loong64": "0.28.2", "@esbuild/linux-mips64el": "0.28.2", "@esbuild/linux-ppc64": "0.28.2", "@esbuild/linux-riscv64": "0.28.2", "@esbuild/linux-s390x": "0.28.2", "@esbuild/linux-x64": "0.28.2", "@esbuild/netbsd-arm64": "0.28.2", "@esbuild/netbsd-x64": "0.28.2", "@esbuild/openbsd-arm64": "0.28.2", "@esbuild/openbsd-x64": "0.28.2", "@esbuild/openharmony-arm64": "0.28.2", "@esbuild/sunos-x64": "0.28.2", "@esbuild/win32-arm64": "0.28.2", "@esbuild/win32-ia32": "0.28.2", "@esbuild/win32-x64": "0.28.2" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA=="], "esm-env": ["esm-env@1.2.2", "", {}, "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA=="], @@ -321,7 +321,7 @@ "windmill-yaml-validator": ["windmill-yaml-validator@1.1.1", "", { "dependencies": { "@stoplight/yaml": "^4.3.0", "ajv": "^8.17.1" } }, "sha512-CVgAwEoBdJhF39q2N012QffhlGPRIyIWd8gj7NnfG+/lMWgH2k5CBLtKIt6cPF8Bxz+6DGC3st1ARSsecDtbTg=="], - "ws": ["ws@8.18.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw=="], + "ws": ["ws@8.21.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw=="], "wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="], diff --git a/cli/package.json b/cli/package.json index 03497dc887..5e7623a7ab 100644 --- a/cli/package.json +++ b/cli/package.json @@ -23,7 +23,7 @@ "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", "@windmill-labs/shared-utils": "^1.0.13", "diff": "^5.2.0", - "esbuild": "0.28.0", + "esbuild": "0.28.2", "get-port": "7.1.0", "jszip": "3.8.0", "minimatch": "^10.0.0", @@ -46,7 +46,7 @@ "windmill-parser-wasm-ts": "1.695.0", "windmill-parser-wasm-yaml": "1.770.0", "windmill-yaml-validator": "1.1.1", - "ws": "8.18.0", + "ws": "8.21.3", "yaml": "^2.7.0" }, "devDependencies": { diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 7772dafedd..0d4ad634c8 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,9 +25,11 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { + guests?: boolean; value: any; public?: boolean; summary: string; @@ -110,6 +116,50 @@ export function replaceInlineScripts( export function isExecutionModeAnonymous(app: any) { return app?.["policy"]?.["execution_mode"] == "anonymous"; } +export function isExecutionModeGuest(app: any) { + return app?.["policy"]?.["execution_mode"] == "guest"; +} +export type AppExecutionMode = "anonymous" | "guest" | "publisher" | "viewer"; +/** The access mode is the one policy field a tracked app keeps, as `public` (anonymous) + * or `guests` (guest); the rest of the policy is preserved from the deployed app on + * push (see `generatingPolicy`). */ +export function markAccessFromPolicy(app: any) { + if (isExecutionModeAnonymous(app)) { + app.public = true; + } else if (isExecutionModeGuest(app)) { + app.guests = true; + } +} +/** The mode the tracked file states, or `undefined` when it states none — the + * normal case, since a pull writes only the two open-access markers. `viewer` + * and `publisher` have no marker of their own, so a file can only name them + * through a policy block it was hand-written with. */ +function statedExecutionMode(app: any): AppExecutionMode | undefined { + if (app?.["public"] ?? isExecutionModeAnonymous(app)) { + return "anonymous"; + } + if (app?.["guests"] ?? isExecutionModeGuest(app)) { + return "guest"; + } + const mode = app?.["policy"]?.["execution_mode"]; + return mode === "viewer" || mode === "publisher" ? mode : undefined; +} + +/** The mode this push deploys under. A file that states one is authoritative, in + * both directions. Otherwise the two open-access markers are all it says, so + * their absence closes a deployed `anonymous`/`guest` app back down to + * `publisher` — while a deployed `viewer` is not a grant those markers revoke, + * so it carries over rather than widening to `publisher`. */ +export function executionModeForPush( + localApp: any, + deployedPolicy: Policy | undefined, +): AppExecutionMode { + const stated = statedExecutionMode(localApp); + if (stated) { + return stated; + } + return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; +} export async function pushApp( workspace: string, remotePath: string, @@ -133,16 +183,11 @@ export async function pushApp( //ignore } - let remoteOnBehalfOf: string | undefined; - let remoteOnBehalfOfEmail: string | undefined; - if (app?.policy) { - remoteOnBehalfOf = app.policy.on_behalf_of; - remoteOnBehalfOfEmail = app.policy.on_behalf_of_email; - } + // `app.policy` is cleared a few lines down, so capture it first: it is the + // base the regenerated policy is built on. + const deployedPolicy: Policy | undefined = app?.policy; - if (isExecutionModeAnonymous(app)) { - app.public = true; - } + markAccessFromPolicy(app); // console.log(app); if (app) { app.policy = undefined; @@ -155,26 +200,19 @@ export async function pushApp( const localApp = (await yamlParseFile(path)) as AppFile; replaceInlineScripts(localApp.value, localPath, true); + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext + ); await generatingPolicy( localApp, remotePath, - localApp?.["public"] ?? - localApp?.["policy"]?.["execution_mode"] == "anonymous" + executionModeForPush(localApp, deployedPolicy), + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) ); - const preserveFields: { preserve_on_behalf_of?: boolean } = {}; - if (permissionedAsContext?.userIsAdminOrDeployer) { - if (app) { - if (localApp.policy && remoteOnBehalfOf) { - (localApp.policy as any).on_behalf_of = remoteOnBehalfOf; - (localApp.policy as any).on_behalf_of_email = remoteOnBehalfOfEmail; - preserveFields.preserve_on_behalf_of = true; - log.info(`Preserving ${remoteOnBehalfOfEmail ?? remoteOnBehalfOf} as permissioned_as for app ${remotePath}`); - } - } - // On create: backend applies folder defaults - } - // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). const { extra_perms: localPerms, ...localAppBody } = localApp as AppFile & { @@ -230,18 +268,76 @@ export async function pushApp( export async function generatingPolicy( app: any, path: string, - publicApp: boolean + executionMode: AppExecutionMode, + base: Policy | undefined ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updatePolicy(app.value, undefined); - app.policy.execution_mode = publicApp ? "anonymous" : "publisher"; + app.policy = await windmillUtils.updatePolicy(app.value, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; } } +/** What the regenerated policy starts from: the deployed one, so a push keeps + * settings the tracked file doesn't record; on a first push, whatever the file + * states. The run identity rides along only when `claimsOnBehalfOf` — never + * from the file, never from a pusher who may not preserve one, since `wmill` + * is regularly pointed at servers older than the rewrite that would fix it. */ +export function basePolicy( + localApp: any, + deployedPolicy: Policy | undefined, + claimsOnBehalfOf: boolean +): Policy | undefined { + const stated = deployedPolicy ?? (localApp?.policy as Policy | undefined); + if (!stated || claimsOnBehalfOf) { + return stated; + } + const base: Policy = { ...stated }; + delete base.on_behalf_of; + delete base.on_behalf_of_email; + return base; +} + +/** Claim the run-as identity the regenerated policy carries over from the + * deployed app. Only a deployed identity may be claimed, never one the tracked + * file states — a repo doesn't get to pick who an app runs as. Without the flag + * the backend rewrites `on_behalf_of` to whoever is pushing, and it only honors + * the flag for an admin or a `wm_deployers` member, so a caller who is neither + * doesn't get to claim it here either. */ +export function preserveOnBehalfOfFields( + remotePath: string, + deployedPolicy: Policy | undefined, + permissionedAsContext: PermissionedAsContext | undefined +): { preserve_on_behalf_of?: boolean } { + const onBehalfOf = deployedPolicy?.on_behalf_of; + if (!permissionedAsContext?.userIsAdminOrDeployer || !onBehalfOf) { + return {}; + } + log.info( + `Preserving ${deployedPolicy?.on_behalf_of_email ?? onBehalfOf} as permissioned_as for app ${remotePath}` + ); + return { preserve_on_behalf_of: true }; +} + +/** The policy is written wholesale by the deploy, so the fields it does not + * derive from the tracked sources have to survive the trip. The policy builder + * has already recomputed what it can — the triggerables on both paths, plus the + * S3 rules on the low-code one, which `updateRawAppPolicy` has no equivalent of + * and so carries over. This sets the two left: the access mode, and the legacy + * `triggerables`, which still grant execution (the backend folds them into + * `triggerables_v2` at run time) and so are dropped rather than carried, or a + * deployed app would keep being able to run runnables this push removed. */ +export function finalizeDerivedPolicy( + policy: Policy, + executionMode: AppExecutionMode +) { + policy.triggerables = undefined; + policy.execution_mode = executionMode; +} + async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -410,10 +506,23 @@ async function push( absoluteFilePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index b45666a1e1..2dac8639e2 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -603,7 +603,7 @@ async function dev(opts: DevOptions, appFolder?: string) { build.onLoad( { filter: /.*/, namespace: "wmill-virtual" }, (args: any) => { - const contents = wmillTs(port); + const contents = wmillTs(); log.info( colors.yellow( `[wmill-virtual] Loading virtual module: ${args.path}`, diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 9f83d0322a..36c8260e25 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -1,6 +1,9 @@ import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; @@ -15,9 +18,24 @@ import { readdir } from "node:fs/promises"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { deepEqual, readTextFile } from "../../utils/utils.ts"; -import { replaceInlineScripts, repopulateFields } from "./app.ts"; +import { + type AppExecutionMode, + basePolicy, + executionModeForPush, + finalizeDerivedPolicy, + markAccessFromPolicy, + preserveOnBehalfOfFields, + replaceInlineScripts, + repopulateFields, +} from "./app.ts"; +import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER, RECORDINGS_FOLDER } from "./app_metadata.ts"; +import { + NEVER_DEPLOYED_DIRS, + NEVER_DEPLOYED_FILES, +} from "../../utils/app_files.ts"; import { writeIfChanged } from "../../utils/utils.ts"; import { yamlOptions } from "../sync/sync.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; @@ -27,6 +45,7 @@ import { } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; export interface AppFile { + guests?: boolean; runnables?: any; custom_path?: string; public?: boolean; @@ -36,6 +55,8 @@ export interface AppFile { tables?: string[]; datatable?: string; schema?: string; + /** The role the app uses each data table through, by data table name. */ + roles?: Record; }; // Mirrors granular ACLs on the raw_app path. Synced via /acls/* by // applyExtraPermsDiff — never through update_app_raw — so a perm-only @@ -309,13 +330,11 @@ async function collectAppFiles( const relativePath = basePath + entry.name; if (entry.isDirectory()) { - // Skip the runnables, node_modules, and sql_to_apply subfolders + // The backend folder deploys as `value.runnables`, not as a bundled + // file; the rest reach the server through no channel at all. if ( entry.name === APP_BACKEND_FOLDER || - entry.name === "node_modules" || - entry.name === "dist" || - entry.name === ".claude" || - entry.name === "sql_to_apply" + NEVER_DEPLOYED_DIRS.has(entry.name) ) { continue; } @@ -327,13 +346,11 @@ async function collectAppFiles( } await readDirRecursive(fullPath + SEP, relativePath + "/"); } else if (entry.isFile()) { - // Skip generated/metadata files that shouldn't be part of the app + // `raw_app.yaml` deploys as the request's metadata rather than as a + // bundled file; the rest reach the server through no channel at all. if ( entry.name === "raw_app.yaml" || - entry.name === "package-lock.json" || - entry.name === "DATATABLES.md" || - entry.name === "AGENTS.md" || - entry.name === "wmill.d.ts" + NEVER_DEPLOYED_FILES.has(entry.name) ) { continue; } @@ -353,6 +370,7 @@ export async function pushRawApp( localPath: string, message?: string, defaultTs: "bun" | "deno" = "bun", + permissionedAsContext?: PermissionedAsContext, ): Promise { if (alreadySynced.includes(localPath)) { return; @@ -369,9 +387,11 @@ export async function pushRawApp( } catch { //ignore } - if (app?.["policy"]?.["execution_mode"] == "anonymous") { - app.public = true; - } + // `app.policy` is cleared a few lines down, so capture it first. `raw_app.yaml` + // records none of the policy, so anything the deploy drawer set is only here. + const deployedPolicy: Policy | undefined = app?.policy; + + markAccessFromPolicy(app); // console.log(app); if (app) { app.policy = undefined; @@ -419,10 +439,21 @@ export async function pushRawApp( // Create a temporary app object for policy generation const appForPolicy = { ...localApp, runnables }; + // On create the backend applies folder defaults, so there is nothing to preserve. + const preserveFields = preserveOnBehalfOfFields( + remotePath, + deployedPolicy, + permissionedAsContext, + ); await generatingPolicy( appForPolicy, remotePath, - localApp?.["public"] ?? false, + executionModeForPush(localApp, deployedPolicy), + basePolicy( + localApp, + deployedPolicy, + !!preserveFields.preserve_on_behalf_of, + ), ); const files = await collectAppFiles(localPath); @@ -477,6 +508,7 @@ export async function pushRawApp( path: remotePath, summary: localApp.summary, policy: appForPolicy.policy, + ...preserveFields, deployment_message: message, // Preserve any user draft at this path (see backend skip_draft_deletion). skip_draft_deletion: true, @@ -526,15 +558,13 @@ export async function pushRawApp( export async function generatingPolicy( app: any, path: string, - publicApp: boolean, + executionMode: AppExecutionMode, + base: Policy | undefined, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { - app.policy = await windmillUtils.updateRawAppPolicy( - app.runnables, - app.policy, - ); - app.policy.execution_mode = publicApp ? "anonymous" : "publisher"; + app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); + finalizeDerivedPolicy(app.policy, executionMode); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -559,6 +589,10 @@ async function pushRawAppCommand( filePath, undefined, merged.defaultTs, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), ); log.info(colors.bold.underline.green("Raw app pushed")); } diff --git a/cli/src/commands/app/wmillTsDev.ts b/cli/src/commands/app/wmillTsDev.ts index 7cb2ea328b..821060d8e4 100644 --- a/cli/src/commands/app/wmillTsDev.ts +++ b/cli/src/commands/app/wmillTsDev.ts @@ -1,5 +1,5 @@ //comment this line and last to dev -export function wmillTsDev(port: number) { return ` +export function wmillTsDev() { return ` let reqs: Record = {} let ws: WebSocket | null = null let wsReady: Promise @@ -10,7 +10,7 @@ function initWebSocket() { wsReadyResolve = resolve }) - ws = new WebSocket('ws://localhost:${port}') + ws = new WebSocket((window.location.protocol === 'https:' ? 'wss:' : 'ws:') + '//' + window.location.host) ws.onopen = () => { console.log('[wmill] WebSocket connected') @@ -157,4 +157,4 @@ export function streamJob( ws?.send(JSON.stringify({ jobId, type: 'streamJob', reqId })) }) } -`} \ No newline at end of file +`} diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 6ace1eaf48..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } @@ -1144,7 +1159,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", @@ -1162,7 +1177,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index 1a1f144de1..1a9c89680e 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -17,6 +17,12 @@ interface HubResourceType { app: string; description: string; is_fileset?: boolean; + // Absent from hubs predating the column, so a missing value is "ordinary type", + // not "unset it". + format_extension?: string | null; + // Null where nobody named the type, and absent from hubs predating the field, which + // leaves a stored name alone rather than clearing it. + display_name?: string | null; } export async function pull(opts: GlobalOptions) { @@ -116,7 +122,10 @@ export async function pull(opts: GlobalOptions) { typeof y.schema !== "string" && deepEqual(y.schema, x.schema) && y.description === x.description && - (y.is_fileset ?? false) === (x.is_fileset ?? false) + (y.is_fileset ?? false) === (x.is_fileset ?? false) && + (y.format_extension ?? null) === (x.format_extension ?? null) && + (x.display_name === undefined || + (y.display_name ?? null) === x.display_name) ) ) { log.info("skipping " + x.name + " (same as current)"); diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index f6a35429ef..d031c16b3f 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -31,10 +31,18 @@ import { import { isFlowInlineScriptPath, isAppInlineScriptPath, - isRawAppPath, + isFolderResourcePathAnyFormat, getFolderSuffix, + getScriptBasePathFromModulePath, } from "../../utils/resource_folders.ts"; -import { exts } from "../script/script.ts"; +import { isFilesetResource } from "../../utils/utils.ts"; +import { + exts, + findContentFile, + hasScriptExt, + isModuleEntryMetadata, + UnresolvableScriptContentFileError, +} from "../script/script.ts"; interface LintOptions extends GlobalOptions { json?: boolean; @@ -67,6 +75,9 @@ export interface LintReport { const YAML_FILE_REGEX = /\.ya?ml$/i; const NATIVE_TRIGGER_REGEX = /\.[^.]+_native_trigger\.ya?ml$/i; +// The metadata suffixes `findContentFile` resolves a flat script from. `.yml` is +// deliberately absent, since the push does not accept it there either. +const FLAT_SCRIPT_METADATA_REGEX = /\.script\.(yaml|json|lock)$/; function normalizePath(p: string): string { return p.replaceAll(SEP, "/"); @@ -643,6 +654,83 @@ export async function checkMissingLocks( return issues; } +/** + * Whether a path is a script's own metadata, as opposed to metadata the push + * deploys through some parent: a folder resource's inline scripts, a fileset's + * children (arbitrarily named, so one may be spelled exactly like a script's + * metadata) and the files of a module or dbt bundle all belong to that parent. + * + * Takes the path as the SYNC ROOT spells it, like the push. Relative to the + * lint target the enclosing folder is gone whenever the target IS that folder; + * absolute, the classifiers match their suffixes ANYWHERE in the string, so a + * checkout under `acme.app` reads as one app and nothing is ever reported. + */ +function isStandaloneScriptMetadata(rootedPath: string): boolean { + // Both suffix formats, because the dotted/non-dotted setting is read from the + // invocation directory and an explicit lint target may not share it. + if ( + isFolderResourcePathAnyFormat(rootedPath) || + isFilesetResource(rootedPath) + ) { + return false; + } + // A module folder keeps its metadata inside itself (`__mod/script.yaml`), + // which is standalone even though every other path under `__mod/` is not. + if (isModuleEntryMetadata(rootedPath)) return true; + if (getScriptBasePathFromModulePath(rootedPath) !== undefined) return false; + return FLAT_SCRIPT_METADATA_REGEX.test(rootedPath); +} + +/** + * `findContentFile` quotes the paths it was given back in its errors, so the + * lint target's own prefix comes off them again. Anchored at a path start: a + * plain substring replace of `f/` also eats the one inside `conf/`, mangling + * the very filename the message is telling the reader to delete. + */ +function relativizeMessage(message: string, prefix: string): string { + if (!prefix) return message; + const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return message.replaceAll(new RegExp(`(^|[\\s(])${escaped}/`, "g"), "$1"); +} + +/** + * Script metadata files that cannot be paired with exactly one content file: + * the push refuses those, and no metadata format makes them deployable, so the + * inactive twin of a format switch (`foo.script.json` in a yaml repo) is dead + * weight worth reporting even though the push skips it rather than refusing it. + * + * Resolved through `findContentFile` rather than by probing `exts` directly, so + * lint and push agree on what counts as paired: a dbt project's descriptor is + * optional and its absence is not an orphan, while two content files beside one + * metadata file is just as undeployable as none. It classifies what it is given + * and looks under `syncRoot`, so where the command was invoked from is not part + * of the answer. + */ +async function checkOrphanScriptMetadata( + syncRoot: string, + prefix: string, + metadataPaths: string[], +): Promise { + const issues: FileIssue[] = []; + for (const metadataPath of metadataPaths) { + const rootedPath = prefix ? `${prefix}/${metadataPath}` : metadataPath; + try { + await findContentFile(rootedPath, syncRoot); + } catch (e) { + if (!(e instanceof UnresolvableScriptContentFileError)) { + log.debug(`Failed to resolve content file for ${rootedPath}: ${e}`); + continue; + } + issues.push({ + path: metadataPath, + target: "script", + errors: [relativizeMessage(e.message, prefix)], + }); + } + } + return issues; +} + export async function runLint( opts: LintOptions, directory?: string, @@ -674,8 +762,16 @@ export async function runLint( const root = await FSFSElement(targetDirectory, [], false); const validator = new WindmillYamlValidator(); + // Walked paths are relative to the lint target; this puts them back the way + // the sync root spells them, which is what the two below are written against. + const syncRoot = await findSyncRoot(targetDirectory); + const metadataPrefix = normalizePath( + path.relative(syncRoot, targetDirectory), + ); + const warnings: LintWarning[] = []; const issues: FileIssue[] = []; + const scriptMetadataPaths: string[] = []; let scannedFiles = 0; let validatedFiles = 0; let validFiles = 0; @@ -689,6 +785,17 @@ export async function runLint( const normalizedPath = normalizePath(entry.path); scannedFiles += 1; + + // Collected before the YAML filter below: `.script.lock` and `.script.json` + // are metadata too, and both fail the push when nothing pairs with them. + if ( + isStandaloneScriptMetadata( + metadataPrefix ? `${metadataPrefix}/${normalizedPath}` : normalizedPath, + ) + ) { + scriptMetadataPaths.push(normalizedPath); + } + if (!YAML_FILE_REGEX.test(normalizedPath)) { continue; } @@ -727,6 +834,16 @@ export async function runLint( } } + // Unconditional: unlike a missing lock, metadata with no content file fails + // every push, so there is no mode in which it is acceptable. + issues.push( + ...(await checkOrphanScriptMetadata( + syncRoot, + metadataPrefix, + scriptMetadataPaths, + )), + ); + // Check for missing locks if --locks-required is set if (opts.locksRequired) { const lockIssues = await checkMissingLocks(opts, explicitTargetDirectory); @@ -820,6 +937,15 @@ async function lint(opts: LintOptions & { watch?: boolean }, directory?: string) } } +/** + * Whether a changed file can change what a lint run reports: metadata in any of + * its formats, and the content files whose presence is what keeps that metadata + * from being an orphan. + */ +function affectsLint(filename: string): boolean { + return /\.(ya?ml|json|lock)$/i.test(filename) || hasScriptExt(filename); +} + async function lintWatch(opts: LintOptions, directory?: string) { const { watch } = await import("node:fs"); const targetDir = directory ? path.resolve(process.cwd(), directory) : process.cwd(); @@ -842,7 +968,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { let debounce: ReturnType | null = null; watch(targetDir, { recursive: true }, (_event, filename) => { - if (!filename || !filename.toString().endsWith(".yaml") && !filename.toString().endsWith(".yml")) return; + if (!filename || !affectsLint(filename.toString())) return; if (debounce) clearTimeout(debounce); debounce = setTimeout(runAndReport, 300); }); @@ -853,7 +979,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { const command = new Command() .description( - "Validate Windmill flow, schedule, and trigger YAML files in a directory", + "Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file", ) .arguments("[directory:string]") .option("--json", "Output results in JSON format") diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 5f9e713f42..f6173a28bb 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -413,7 +413,13 @@ export function parseMuteAnnotations(content: string): { // Comment prefix for `volume:` annotations. Deliberately NOT `commentPrefix` // above (which returns `--` for SQL): volume annotations are only recognized for // the languages the backend/frontend recognize them for — mirrors -// `asset_inference.rs:comment_prefix` and `infer.ts:getCommentPrefix` (SQL → none). +// `asset_inference.rs:comment_prefix` and `infer.ts:getCommentPrefix` (SQL → none), +// minus `php`. Those two recognize PHP, but a PHP script cannot reach this map: it +// has no wasm asset parser, so `fallbackParse` handles it and that scan breaks on +// the mandatory ` { path = removeType(path, "schedule").replaceAll(SEP, "/"); log.debug(`Processing local schedule ${path}`); @@ -123,6 +130,21 @@ export async function pushSchedule( // Strip CLI-only boolean marker before sending to API delete (localSchedule as any).has_permissioned_as; + // In a fork, the file's `enabled` is the parent's for a path the parent + // also has (see sync push's `parentOwnedScheduleEnabled`): the fork's own + // flag stays as it is. + if (enabledOwnedByParent && schedule) { + if ( + localSchedule.enabled !== undefined && + localSchedule.enabled !== schedule.enabled + ) { + log.warnAlways( + `Schedule ${path} stays ${schedule.enabled ? "enabled" : "disabled"}: the file says ${localSchedule.enabled ? "enabled" : "disabled"}, but in a fork that flag is the parent workspace's` + ); + } + delete localSchedule.enabled; + } + const preserveFields: { permissioned_as?: string; preserve_permissioned_as?: boolean } = {}; if (permissionedAsContext?.userIsAdminOrDeployer) { if (schedule) { @@ -153,13 +175,9 @@ export async function pushSchedule( ...preserveFields, }, }); - // Tarball export from a fork strips `enabled` from schedule YAMLs so - // the fork→parent git-sync round-trip can't flip the parent's state. - // Skip the secondary setScheduleEnabled call when the local YAML - // doesn't carry `enabled` — sending `{ enabled: undefined }` would - // serialize to `{}` and the backend (`SetEnabled.enabled` is required) - // would reject the request. Preserving the target's existing flag is - // exactly the round-trip-safe behavior. + // No `enabled` in the file (absent from the YAML, or set aside above) + // leaves the remote flag alone: `SetEnabled.enabled` is required, so + // `{ enabled: undefined }` would be rejected rather than ignored. if ( localSchedule.enabled !== undefined && localSchedule.enabled !== schedule.enabled @@ -167,13 +185,12 @@ export async function pushSchedule( log.info(colors.bold.yellow( `Schedule ${path} is ${localSchedule.enabled ? "enabled" : "disabled"} locally but not on remote, updating remote` )); - await wmill.setScheduleEnabled({ - workspace: workspace, + await setEnabledUnlessParentOwned( + workspace, path, - requestBody: { - enabled: localSchedule.enabled, - }, - }); + localSchedule.enabled, + schedule.enabled + ); } } catch (e) { console.error((e as any).body); @@ -194,6 +211,44 @@ export async function pushSchedule( console.error((e as any).body); throw e; } + // A create in a fork lands disabled whatever the request says. A fork-only + // path the file wants enabled is enabled here, so one push converges; a + // parent-owned one stays disabled. + if (enabledOwnedByParent !== undefined && localSchedule.enabled === true) { + if (enabledOwnedByParent) { + log.warnAlways( + `Schedule ${path} created disabled: the file says enabled, but in a fork that flag is the parent workspace's` + ); + } else { + await setEnabledUnlessParentOwned(workspace, path, true, false); + } + } + } +} + +// The parent listing behind `enabledOwnedByParent` sees only what the pusher +// may read; the backend's `fork-conflict` refusal is the last word, so a path +// it says the parent has keeps the fork's flag rather than failing the push. +async function setEnabledUnlessParentOwned( + workspace: string, + path: string, + enabled: boolean, + remoteEnabled: boolean +): Promise { + try { + await wmill.setScheduleEnabled({ + workspace, + path, + requestBody: { enabled }, + }); + } catch (e) { + const conflict = parseForkConflict(e); + if (!conflict) { + throw e; + } + log.warnAlways( + `Schedule ${path} left ${remoteEnabled ? "enabled" : "disabled"}: the parent workspace '${conflict.parentWorkspaceId}' has the same schedule, so its flag is the parent's to set` + ); } } @@ -250,8 +305,12 @@ async function disable(opts: GlobalOptions, path: string) { } async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); if (!validatePath(remotePath)) { return; @@ -268,7 +327,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { workspace.workspaceId, remotePath, undefined, - parseFromFile(filePath) + parseFromFile(filePath), + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) ); console.log(colors.bold.underline.green("Schedule pushed")); } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 8c93bdb963..fe193e5d85 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -7,6 +7,7 @@ import { validatePath, } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import { writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; @@ -58,6 +59,7 @@ import { SyncOptions, mergeConfigWithConfigFile, readConfigFile, + readEffectiveSyncBehavior, } from "../../core/conf.ts"; import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; @@ -231,7 +233,11 @@ async function push(opts: PushOptions, filePath: string) { opts.message, opts, await getRawWorkspaceDependencies(true), - codebases + codebases, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace) + ) ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); } @@ -747,14 +753,10 @@ export async function handleFile( // create_script (which would bump the script hash) and instead route // through /acls/* via applyExtraPermsDiff. // - // No refetch is needed: - // - folder perms are additive at auth time, never merged onto item rows; - // - the body sent to create_script doesn't carry extra_perms, so a fresh - // deploy of an existing path inherits the previous version's perms - // unchanged. The diff against `remote` (captured before the deploy) - // therefore matches what `wmill acl remove` would do — and the granular - // ACL endpoint updates every matching row, so the inheritance on the - // new version doesn't leave ghost entries. + // No refetch is needed: folder perms are additive at auth time and never merged + // onto item rows, and each branch above leaves the new version's perms where the + // diff expects them — the update branch names a parent, which carries them over, + // while the create branch has no `remote` to diff against and sends the whole set. await applyExtraPermsDiff( workspaceId, "script", @@ -1073,10 +1075,12 @@ export class DbtPathCollisionError extends UnresolvableScriptContentFileError {} * guard on one of them leaves the other silently overwriting. */ export async function collidingDbtProject( - basePath: string + basePath: string, + baseDir?: string ): Promise { const project = basePath + "__dbt/dbt_project.yml"; - return (await stat(project).then(() => true).catch(() => false)) + const onDisk = baseDir ? path.join(baseDir, project) : project; + return (await stat(onDisk).then(() => true).catch(() => false)) ? project : undefined; } @@ -1137,7 +1141,14 @@ async function readScriptContent(filePath: string): Promise { } } -export async function findContentFile(filePath: string) { +/** + * The script file `filePath`'s metadata belongs to. `baseDir`, when given, is + * where the disk lookups happen, leaving `filePath` classified as written: the + * layout helpers below match their suffixes ANYWHERE in a path, so a caller + * that prefixed a checkout named `repo__mod` would have it read as the module. + */ +export async function findContentFile(filePath: string, baseDir?: string) { + const onDisk = (p: string) => (baseDir ? path.join(baseDir, p) : p); // Folder layout: __mod/script.yaml -> __mod/script.ts const isModuleFolderMeta = isModuleEntryMetadata(filePath); const toCandidate = (ext: string) => @@ -1161,7 +1172,7 @@ export async function findContentFile(filePath: string) { const validCandidates = ( await Promise.all( candidates.map((x) => { - return stat(x) + return stat(onDisk(x)) .catch(() => undefined) .then((x) => x?.isFile()) .then((e) => { @@ -1181,6 +1192,7 @@ export async function findContentFile(filePath: string) { const dbtCandidate = toCandidate("__dbt/" + DBT_DESCRIPTOR_NAME); const dbtProject = await collidingDbtProject( dbtCandidate.slice(0, -("__dbt/" + DBT_DESCRIPTOR_NAME).length), + baseDir, ); const nonDbtCandidates = validCandidates.filter((c) => c !== dbtCandidate); if (dbtProject && nonDbtCandidates.length > 0) { @@ -2213,7 +2225,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", @@ -2231,7 +2243,7 @@ const command = new Command() .arguments("") .option( "-d --data ", - "Inputs specified as a JSON string or a file using @ or stdin using @-." + "Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path." ) .option( "-s --silent", diff --git a/cli/src/commands/shared_ui.ts b/cli/src/commands/shared_ui.ts index 3c01870fd1..f8e2965994 100644 --- a/cli/src/commands/shared_ui.ts +++ b/cli/src/commands/shared_ui.ts @@ -26,37 +26,33 @@ async function readDirRecursive( return out; } +/** The workspace's shared UI store, or undefined when it cannot be read. */ +async function fetchSharedUi( + workspace: string, +): Promise | undefined> { + try { + const got = await wmill.getSharedUi({ workspace }); + return got.files ?? {}; + } catch { + return undefined; + } +} + export type SharedUiChange = | { type: "added"; path: string } | { type: "edited"; path: string; before: string; after: string } | { type: "deleted"; path: string }; /** - * Diff the local /ui/ folder against the workspace's shared UI store in - * the push direction (local -> remote), returning entries whose `path` is - * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the - * dry-run preview and the real push never diverge. - * - * Mirrors pushSharedUi's no-op: with no local ui/ folder there is nothing to - * push, so the apply is a no-op and the preview must be empty (even when the - * remote store is non-empty) to avoid phantom diffs the apply won't perform. + * The push-direction diff (local -> remote) of two already-read file maps. + * Split out so the preview and the push itself compare identically off one + * read of each side. */ -export async function diffSharedUi(workspace: string): Promise { - const localDir = path.join(process.cwd(), SHARED_UI_DIR); - if (!fs.existsSync(localDir)) { - return []; - } - const files = await readDirRecursive(localDir); - - let remote: Record = {}; - try { - const got = await wmill.getSharedUi({ workspace }); - remote = got.files ?? {}; - } catch { - // If endpoint missing or unauthorized, treat remote as empty (the push - // would attempt the PUT anyway). - } - +function sharedUiChanges( + files: Record, + remote: Record, + keepDeleted?: boolean, +): SharedUiChange[] { // Use Object.hasOwn, not `in`: a file named after an Object.prototype member // (e.g. ui/toString) would otherwise register as always-present and be // misdiffed. @@ -69,34 +65,97 @@ export async function diffSharedUi(workspace: string): Promise changes.push({ type: "edited", path: p, before: remote[rel], after: content }); } } - for (const rel of Object.keys(remote)) { - if (!Object.hasOwn(files, rel)) { - changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` }); + if (!keepDeleted) { + for (const rel of Object.keys(remote)) { + if (!Object.hasOwn(files, rel)) { + changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` }); + } } } return changes; } +/** + * Diff the local /ui/ folder against the workspace's shared UI store in + * the push direction (local -> remote), returning entries whose `path` is + * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the + * dry-run preview and the real push never diverge. + * + * Mirrors pushSharedUi's no-ops: with no local ui/ folder, or under + * `keepDeleted` with an unreadable store, the apply does nothing, so the + * preview must be empty (even when the remote store is non-empty) to avoid + * phantom diffs the apply won't perform. + */ +export async function diffSharedUi( + workspace: string, + keepDeleted?: boolean, +): Promise { + const localDir = path.join(process.cwd(), SHARED_UI_DIR); + if (!fs.existsSync(localDir)) { + return []; + } + const files = await readDirRecursive(localDir); + const remote = await fetchSharedUi(workspace); + if (remote === undefined && keepDeleted) { + return []; + } + // If endpoint missing or unauthorized, treat remote as empty (the push + // would attempt the PUT anyway). + return sharedUiChanges(files, remote ?? {}, keepDeleted); +} + /** * Push the local /ui/ folder to the workspace's shared UI store. * Returns true if a push was performed, false if the folder is missing or * already matches the remote store. Note an empty-but-existing folder still - * pushes an empty map (clearing the remote store) if the remote is non-empty. + * pushes an empty map (clearing the remote store) if the remote is non-empty — + * unless `keepDeleted`, which folds remote-only files back into the map. */ -export async function pushSharedUi(workspace: string): Promise { +export async function pushSharedUi( + workspace: string, + keepDeleted?: boolean, +): Promise { const localDir = path.join(process.cwd(), SHARED_UI_DIR); if (!fs.existsSync(localDir)) { return false; } - // Skip if no change — reuse diffSharedUi so preview and push never diverge. - const diff = await diffSharedUi(workspace); - if (diff.length === 0) { + const files = await readDirRecursive(localDir); + const remote = await fetchSharedUi(workspace); + // The store is written whole, so a remote-only file is pruned by omission + // alone. An unreadable store leaves no way to carry those files over, so + // the shared UI is left untouched rather than cleared. + if (remote === undefined && keepDeleted) { + log.warn( + colors.yellow( + "Could not read the shared UI folder from the remote; skipping its push so --keep-deleted does not clear it.", + ), + ); + return false; + } + + // Same comparison as diffSharedUi, off the same two maps, so preview and + // push never diverge. + if (sharedUiChanges(files, remote ?? {}, keepDeleted).length === 0) { log.info(colors.gray("Shared UI folder up to date")); return false; } - const files = await readDirRecursive(localDir); + if (keepDeleted) { + for (const [rel, content] of Object.entries(remote!)) { + // defineProperty, not assignment: `files.__proto__ = "..."` runs the + // inherited setter and creates no own property, so a remote `ui/__proto__` + // would be dropped from the whole-map PUT — i.e. deleted despite the flag. + if (!Object.hasOwn(files, rel)) { + Object.defineProperty(files, rel, { + value: content, + writable: true, + enumerable: true, + configurable: true, + }); + } + } + } await wmill.updateSharedUi({ workspace, requestBody: { files }, @@ -111,9 +170,12 @@ export async function pushSharedUi(workspace: string): Promise { /** * Pull the workspace's shared UI store into /ui/. - * Files removed remotely are also removed locally. + * Files removed remotely are also removed locally, unless `keepDeleted`. */ -export async function pullSharedUi(workspace: string): Promise { +export async function pullSharedUi( + workspace: string, + keepDeleted?: boolean, +): Promise { const localDir = path.join(process.cwd(), SHARED_UI_DIR); let got; try { @@ -145,15 +207,17 @@ export async function pullSharedUi(workspace: string): Promise { } // Delete locally-orphaned files - const known = new Set(Object.keys(files)); - const local = await readDirRecursive(localDir); - for (const rel of Object.keys(local)) { - if (!known.has(rel)) { - const full = path.join(localDir, rel); - try { - fs.unlinkSync(full); - } catch { - // ignore + if (!keepDeleted) { + const known = new Set(Object.keys(files)); + const local = await readDirRecursive(localDir); + for (const rel of Object.keys(local)) { + if (!known.has(rel)) { + const full = path.join(localDir, rel); + try { + fs.unlinkSync(full); + } catch { + // ignore + } } } } diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index b1fca78ed2..32cff4a3cb 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -25,7 +25,7 @@ import { } from "yaml"; import JSZip from "jszip"; import { minimatch } from "minimatch"; -import { yamlParseContent } from "../../utils/yaml.ts"; +import { yamlParseContent, yamlParseFile } from "../../utils/yaml.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -124,6 +128,8 @@ import { } from "../../utils/metadata.ts"; import { DoubleLinkedDependencyTree, + LocalScripts, + resolvePlaceholdersFromLocal, uploadScripts, } from "../../utils/dependency_tree.ts"; import { @@ -142,12 +148,13 @@ import { extractCurrentMapping, } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; -import { isExecutionModeAnonymous } from "../app/app.ts"; +import { markAccessFromPolicy } from "../app/app.ts"; import { APP_BACKEND_FOLDER, generateAppLocksInternal, RECORDINGS_FOLDER, } from "../app/app_metadata.ts"; +import { deploysWithRawApp } from "../../utils/app_files.ts"; import { isFlowPath, isAppPath, @@ -171,6 +178,7 @@ import { isDbtModulePath, isDbtGeneratedPath, isModuleEntryPoint, + scriptPathToRemotePath, getScriptBasePathFromModulePath, hasWrongFormatSuffix, DBT_DESCRIPTOR_NAME, @@ -429,37 +437,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +484,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -1138,7 +1088,7 @@ export function rawAppPathWithinFolder( return resolved; } -function ZipFSElement( +export function ZipFSElement( zip: JSZip, useYaml: boolean, defaultTs: "bun" | "deno", @@ -1146,6 +1096,14 @@ function ZipFSElement( resourceTypeToIsFileset: Record, ignoreCodebaseChanges: boolean, stripOnBehalfOf: boolean, + // Names a flow's rendered inline-script files after the checkout's own + // `!inline` references (module id -> file). The export carries script + // source, never a reference, so without a checkout to defer to every file + // is named from the step summary, and a file the checkout names otherwise + // reads as a delete + add on every push while the resolved flows are equal. + localFlowInlineMapping?: ( + flowDir: string, + ) => Promise>, ): DynFSElement { // Pre-scan: find zip base paths of scripts that have modules. // These scripts use the folder layout: {basePath}__mod/script.{ext} @@ -1249,59 +1207,71 @@ function ZipFSElement( log.error(`Failed to parse flow.yaml at path: ${p}`); throw error; } - let inlineScripts; + let inlineScripts: InlineScript[]; try { - const assigner = newPathAssigner(defaultTs, { - skipInlineScriptSuffix: getNonDottedPaths(), - }); - // Preserve original !inline filenames from the flow to avoid phantom renames - const inlineMapping = extractCurrentMapping( - flow.value.modules as any, - {}, - flow.value.failure_module, - flow.value.preprocessor_module, - ); - inlineScripts = extractInlineScriptsForFlows( - flow.value.modules as any, - inlineMapping, - SEP, - defaultTs, - assigner, - { + // Extraction rewrites the modules' content into `!inline` refs, + // so each attempt works on its own copy of the flow. + const render = ( + source: OpenFlow, + inlineMapping: Record, + ): [OpenFlow, InlineScript[]] => { + const f: OpenFlow = structuredClone(source); + const assigner = newPathAssigner(defaultTs, { + skipInlineScriptSuffix: getNonDottedPaths(), + }); + const options = { skipInlineScriptSuffix: getNonDottedPaths(), failOnInlineDirective: true, - }, - ); - if (flow.value.failure_module) { - inlineScripts.push( - ...extractInlineScriptsForFlows( - [flow.value.failure_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { - skipInlineScriptSuffix: getNonDottedPaths(), - failOnInlineDirective: true, - }, - ), - ); - } - if (flow.value.preprocessor_module) { - inlineScripts.push( - ...extractInlineScriptsForFlows( - [flow.value.preprocessor_module], - inlineMapping, - SEP, - defaultTs, - assigner, - { - skipInlineScriptSuffix: getNonDottedPaths(), - failOnInlineDirective: true, - }, - ), + }; + const scripts = extractInlineScriptsForFlows( + f.value.modules as any, + inlineMapping, + SEP, + defaultTs, + assigner, + options, ); + if (f.value.failure_module) { + scripts.push( + ...extractInlineScriptsForFlows( + [f.value.failure_module], + inlineMapping, + SEP, + defaultTs, + assigner, + options, + ), + ); + } + if (f.value.preprocessor_module) { + scripts.push( + ...extractInlineScriptsForFlows( + [f.value.preprocessor_module], + inlineMapping, + SEP, + defaultTs, + assigner, + options, + ), + ); + } + return [f, scripts]; + }; + const inlineMapping = localFlowInlineMapping + ? await localFlowInlineMapping(finalPath) + : {}; + let rendered = render(flow, inlineMapping); + // The assigner keeps the names it hands out unique, not the + // checkout's: one of those equal to another step's + // summary-derived name would leave two files at one path, so + // such a flow renders the export's way. + if ( + new Set(rendered[1].map((s) => s.path)).size !== + rendered[1].length + ) { + rendered = render(flow, {}); } + [flow, inlineScripts] = rendered; } catch (error) { log.error( `Failed to extract inline scripts for flow at path: ${p}`, @@ -1373,9 +1343,7 @@ function ZipFSElement( }; } - if (isExecutionModeAnonymous(app)) { - app.public = true; - } + markAccessFromPolicy(app); app.policy = undefined; yield { isDirectory: false, @@ -1393,9 +1361,7 @@ function ZipFSElement( log.error(`Failed to parse app.yaml at path: ${p}`); throw error; } - if (rawApp?.["policy"]?.["execution_mode"] == "anonymous") { - rawApp.public = true; - } + markAccessFromPolicy(rawApp); // console.log("rawApp", rawApp); rawApp.policy = undefined; // custom_path is derived from the file path, don't store it @@ -2056,20 +2022,18 @@ export async function elementsToMap( } if (isRawAppFile(path)) { - // FSFSElement builds paths with the platform separator, while the checks - // below are written with "/": without normalizing, none of them match on - // Windows and the push collector's own exclusions become perpetual diffs. + // FSFSElement builds paths with the platform separator, while + // `deploysWithRawApp` is written with "/": without normalizing it matches + // nothing on Windows and the push collector's own exclusions become + // perpetual diffs. const suffix = path .split(getFolderSuffix("raw_app") + SEP) .pop() ?.replaceAll(SEP, "/"); - if ( - suffix?.startsWith("dist/") || - suffix?.startsWith(RECORDINGS_FOLDER + "/") || - suffix == "wmill.d.ts" || - suffix == "package-lock.json" || - suffix == "DATATABLES.md" - ) { + // A file no push sends is not a change to track. Listing it leaves it + // pending forever — nothing ever uploads it — and pushing it redeploys + // the whole app, reassigning its run-as user, to ship nothing. + if (suffix && !deploysWithRawApp(suffix)) { continue; } } @@ -2581,7 +2545,36 @@ export function preservePendingScriptLocks( } } -async function compareDynFSElement( +// `sync push` never applies the workspace's display name from settings.yaml and +// applies its color and auto_invite.instance_groups only when the local file +// carries them (see pushWorkspaceSettings), so on a push the fields it would not +// apply must compare equal, or the row is listed on every run. +const isWorkspaceSettingsFile = (p: string) => + /^settings(\.[^./\\]+)?\.(yaml|json)$/.test(p); +function stripUnappliedSettingsFields(local: any, remote: any) { + delete local?.name; + delete remote?.name; + if (local?.color == null) { + delete local?.color; + delete remote?.color; + } + // push reads a missing auto_invite as {} on both sides + if (local) local.auto_invite ??= {}; + if (remote) remote.auto_invite ??= {}; + const localInvite = local?.auto_invite; + const remoteInvite = remote?.auto_invite; + if (localInvite?.instance_groups == null) { + for (const invite of [localInvite, remoteInvite]) { + delete invite?.instance_groups; + delete invite?.instance_groups_roles; + } + } else { + localInvite.instance_groups_roles ??= {}; + if (remoteInvite) remoteInvite.instance_groups_roles ??= {}; + } +} + +export async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, ignore: (path: string, isDirectory: boolean) => boolean, @@ -2594,6 +2587,9 @@ async function compareDynFSElement( branchOverride?: string, isEls1Remote?: boolean, caseInsensitiveFs?: boolean, + // Which schedule files carry an `enabled` that is not the target's to set + // (see push's `parentOwnedScheduleEnabled`): those compare without it. + parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, ): Promise<{ changes: Change[]; localMap: Record }> { let [m1, m2] = els2 ? await Promise.all([ @@ -2785,12 +2781,34 @@ async function compareDynFSElement( ); throw error; } + if ( + parentOwnsScheduleEnabled && + getTypeStrFromPath(k) === "schedule" && + parentOwnsScheduleEnabled(k) + ) { + delete parsedV?.enabled; + delete parsedM2?.enabled; + } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(parsedV, parsedM2); + } if (deepEqual(parsedV, parsedM2)) { continue; } } else if (k.endsWith(".yaml")) { const before = parseYaml(k, m2[k]); const after = parseYaml(k, v); + if ( + parentOwnsScheduleEnabled && + getTypeStrFromPath(k) === "schedule" && + parentOwnsScheduleEnabled(k) + ) { + delete before?.enabled; + delete after?.enabled; + } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(after, before); + } if (deepEqual(before, after)) { continue; } @@ -3174,6 +3192,104 @@ export function untrackedDatatableMigrationDeletions< ); } +/** + * The kind of secret-bearing object whose deletion this path is, if it is one. + * + * Classified with the push's own `getTypeStrFromPath`, so this agrees with the switch + * that does the deleting. A fileset child is excluded ahead of it: it can be any file, + * `inner.resource.yaml` included, and deleting one re-pushes the parent resource + * rather than deleting anything. + */ +export function secretBearingObjectKind( + p: string, +): "variable" | "resource" | undefined { + if (isFilesetResource(p)) return undefined; + // The apply loop `continue`s past a `.lock` deletion before reaching the switch, + // so counting one would announce a deletion the push never performs. A raw-app or + // dbt `.lock`, the two that loop does not skip, classifies as its bundle's own kind + // long before the file-resource check, so a plain suffix test is enough here. + if (p.endsWith(".lock")) return undefined; + let typ: string; + try { + typ = getTypeStrFromPath(p); + } catch { + // Not a path the push classifies, so not one it deletes. + return undefined; + } + return typ === "variable" || typ === "resource" ? typ : undefined; +} + +/** The server-side object a secret-bearing file belongs to, so a file resource's two + * files are counted (and reported) as the one resource they delete. */ +function secretBearingObjectPath(p: string): string { + const normalized = p.replaceAll(SEP, "/"); + return secretBearingObjectKind(p) === "resource" + ? removeResourceSuffix(normalized) + : normalized.replace(/\.variable\.(yaml|json)$/, ""); +} + +/** e.g. "2 variables and 1 resource", counted by object rather than by file. */ +export function describeSecretBearingChanges( + changes: { path: string }[], +): string { + const objects = { variable: new Set(), resource: new Set() }; + for (const c of changes) { + const kind = secretBearingObjectKind(c.path); + if (kind) objects[kind].add(secretBearingObjectPath(c.path)); + } + return (["variable", "resource"] as const) + .filter((k) => objects[k].size > 0) + .map((k) => `${objects[k].size} ${k}${objects[k].size > 1 ? "s" : ""}`) + .join(" and "); +} + +/** + * Whether a pull change removes a local dbt descriptor. A dbt project's + * descriptor is optional and the remote spells "this project names none" as + * empty content, so its removal reaches the apply loop as an add or an edit + * whose content is `""` — a `deleted` change is never produced for it. + */ +function removesDbtDescriptor(change: Change): boolean { + return ( + isDbtDescriptorPath(change.path) && + ((change.name === "added" && change.content === "") || + (change.name === "edited" && change.after === "")) + ); +} + +/** + * `--keep-deleted`: strip every deletion from the changeset, in place, so the + * sync only adds and updates. A path missing on one side is not on its own + * evidence that it should go from the other — a partial clone, a scoped + * checkout or an item authored in the UI all read as deletions here. + * + * A dbt descriptor the remote no longer names counts too, but only on a pull, + * where applying it removes a local file — `added` as much as `edited`, since a + * stateful pull compares `.wmill` and absence from that map says nothing about + * the working tree the apply loop deletes from. A push removes nothing: the + * descriptor is a script's content, so an empty one updates the script in place. + */ +export function dropDeletions( + changes: Change[], + keptOn: "local" | "remote", +): void { + const isDeletion = (c: Change) => + c.name === "deleted" || (keptOn === "local" && removesDbtDescriptor(c)); + const deletions = changes.filter(isDeletion); + if (deletions.length === 0) return; + const kept = changes.filter((c) => !isDeletion(c)); + changes.length = 0; + changes.push(...kept); + log.info( + colors.yellow( + `--keep-deleted: keeping ${deletions.length} item(s) that exist only ` + + (keptOn === "local" + ? `on disk instead of deleting them locally` + : `on the remote instead of deleting them from the workspace`), + ), + ); +} + interface ChangeTracker { scripts: string[]; flows: string[]; @@ -3286,6 +3402,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { } } +/** + * Index the checkout's standalone scripts by the remote path a relative import + * resolves to, reusing the content the local/remote diff already read. + * + * Same classification as `addToChangedIfNotExists`: a flow or app inline script + * is not addressable as an import target, and a module bundle is addressed by + * its entry point. + */ +function localScriptsByRemotePath( + localMap: Record, +): LocalScripts { + const byRemotePath: LocalScripts = new Map(); + for (const [p, content] of Object.entries(localMap)) { + if (isScriptModulePath(p)) { + if (!isModuleEntryPoint(p)) continue; + } else if ( + !hasScriptExt(p) || + isDatatableMigrationPath(p) || + isFileResource(p) || + isFilesetResource(p) || + isFlowPath(p) || + isAppPath(p) || + isRawAppPath(p) + ) { + continue; + } + byRemotePath.set(scriptPathToRemotePath(p), { localPath: p, content }); + } + return byRemotePath; +} + export async function buildTracker(changes: Change[]) { const tracker: ChangeTracker = { scripts: [], @@ -3433,6 +3580,7 @@ export async function pull( repository?: string; promotion?: string; branch?: string; + keepDeleted?: boolean; useIndividualBranch?: boolean; groupByFolder?: boolean; gitDeployItems?: string; @@ -3446,12 +3594,15 @@ export async function pull( ) { if ((opts as any).jsonOutput) log.setSilent(true); const originalCliOpts = { ...opts }; - opts = await mergeConfigWithConfigFile(opts); - - // --include-secrets overrides skipSecrets from wmill.yaml - if ((originalCliOpts as any).includeSecrets) { - opts.skipSecrets = false; - } + const withConfigFile = async () => { + const merged = await mergeConfigWithConfigFile({ ...originalCliOpts }); + // --include-secrets overrides skipSecrets from wmill.yaml + if ((originalCliOpts as any).includeSecrets) { + merged.skipSecrets = false; + } + return merged; + }; + opts = await withConfigFile(); // Resolve workspace name for config lookups. // --branch resolves git branch → workspace name (deprecated but still supported). @@ -3486,10 +3637,6 @@ export async function pull( throw error; } - if (opts.stateful) { - await mkdir(path.join(process.cwd(), ".wmill"), { recursive: true }); - } - const workspace = await resolveWorkspace(opts, wsNameForConfig); await requireLogin(opts); @@ -3581,6 +3728,14 @@ export async function pull( }); return; } + + // The pull writes into the branch now checked out, so its wmill.yaml + // applies, not the cloned branch's: a fork branch that turned on + // `dedupeLockfiles` would otherwise get one lockfile per script back. + if (getCurrentGitBranch() !== clonedBranchName) { + opts = await withConfigFile(); + wsNameForConfig = resolveWsNameForConfigFromFlags(opts); + } } // If wsNameForConfig wasn't set from flags, infer from the resolved profile @@ -3615,6 +3770,10 @@ export async function pull( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + if (opts.stateful) { + await mkdir(path.join(process.cwd(), ".wmill"), { recursive: true }); + } + const codebases = await listSyncCodebases(opts); log.info( @@ -3684,6 +3843,10 @@ export async function pull( await isCaseInsensitiveFilesystem(process.cwd()), ); + if (opts.keepDeleted) { + dropDeletions(changes, "local"); + } + log.info( `remote (${workspace.name}) -> local: ${changes.length} changes to apply`, ); @@ -3781,22 +3944,17 @@ export async function pull( const target = path.join(process.cwd(), targetPath); const stateTarget = path.join(process.cwd(), ".wmill", targetPath); - // An empty dbt descriptor is not a file: the remote spells "this project - // named no descriptor" as empty content, and writing that would put a - // Windmill file inside a project that has none. ABSENCE is the state to - // reach, so both copies are removed if present and their being missing — - // a project pulled for the first time — is the goal, not an error. The - // `.wmill` copy goes too, or the same change is reported on every pull. + // ABSENCE is the state to reach, never an empty file: writing one would + // put a Windmill file inside a project that has none. So both copies are + // removed if present, and their being missing — a project pulled for the + // first time — is the goal, not an error. The `.wmill` copy goes too, or + // the same change is reported on every pull. // // `force` covers the missing file and NOTHING else: a permission or // read-only-filesystem failure has to surface, or the pull reports // success while the old descriptor — its warehouse, its command, its // arguments — is still what runs locally. - if ( - isDbtDescriptorPath(change.path) && - ((change.name === "added" && change.content === "") || - (change.name === "edited" && change.after === "")) - ) { + if (removesDbtDescriptor(change)) { await rm(target, { force: true }); if (opts.stateful) { await rm(stateTarget, { force: true }); @@ -4117,10 +4275,14 @@ export async function pull( } } - try { - await pullSharedUi(workspace.workspaceId); - } catch (e) { - log.warn(`Failed to pull shared UI folder: ${e}`); + // Skipped under --dry-run since pullSharedUi writes to the local ui/ folder. + // An empty changeset falls through the return above and reaches here. + if (!opts.dryRun) { + try { + await pullSharedUi(workspace.workspaceId, opts.keepDeleted); + } catch (e) { + log.warn(`Failed to pull shared UI folder: ${e}`); + } } // Datatable migrations are part of the workspace export now, so they flow @@ -4425,12 +4587,19 @@ function removeSuffix(str: string, suffix: string) { } // Shown after a `wmill sync push --dry-run` preview that has changes. `sync push` -// deploys to the remote workspace and is destructive (it overwrites and prunes -// remote items that differ from or are absent locally), so the preview reminds -// the caller — especially an AI agent that ran the dry-run to inspect changes — -// to get explicit user confirmation before applying it for real. -const SYNC_PUSH_DESTRUCTIVE_WARNING = - "`wmill sync push` is destructive: applying it deploys these changes to the remote workspace and overwrites or deletes remote items that differ from or are absent locally — this is not automatically reversible. If you are an AI agent, do NOT run `wmill sync push` (without --dry-run) until the user has explicitly confirmed this deploy, unless your custom instructions explicitly allow bypassing that confirmation."; +// deploys to the remote workspace and is destructive (it overwrites remote items +// that differ from local, and prunes those absent locally unless --keep-deleted), +// so the preview reminds the caller — especially an AI agent that ran the dry-run +// to inspect changes — to get explicit user confirmation before applying it for real. +function syncPushDestructiveWarning(keepDeleted?: boolean): string { + return ( + "`wmill sync push` is destructive: applying it deploys these changes to the remote workspace and overwrites " + + (keepDeleted + ? "remote items that differ from local" + : "or deletes remote items that differ from or are absent locally") + + " — this is not automatically reversible. If you are an AI agent, do NOT run `wmill sync push` (without --dry-run) until the user has explicitly confirmed this deploy, unless your custom instructions explicitly allow bypassing that confirmation." + ); +} // A script pushed without a local lock queues a server-side dependency job; if // that job fails the script deploys broken (no lock/assets) with no CLI signal. @@ -4485,11 +4654,92 @@ async function checkServerLockJobs( } } +// The checkout's `!inline` references of one flow (module id -> file), as +// `ZipFSElement`'s `localFlowInlineMapping` names the remote render. Empty +// when the flow has no local flow.yaml. +export async function checkoutInlineNames( + flowYamlPath: string, +): Promise> { + let flow: any; + try { + flow = await yamlParseFile(flowYamlPath); + } catch { + return {}; + } + const mapping = extractCurrentMapping( + flow?.value?.modules, + {}, + flow?.value?.failure_module, + flow?.value?.preprocessor_module, + ); + // A reference that leaves the flow folder would render the remote step + // onto another item's path; such a step keeps its summary-derived name. + for (const [id, ref] of Object.entries(mapping)) { + if (path.isAbsolute(ref) || ref.split(/[\\/]/).includes("..")) { + delete mapping[id]; + } + } + return mapping; +} + +// For a path the parent also has, a fork's export writes the parent's +// `enabled` and the backend refuses to enable the fork's copy: the file's flag +// is the parent's. A parent that cannot be listed (a fork-scoped job token) +// may own every path. Undefined when the target is not a fork. +async function parentOwnedScheduleEnabled( + workspaceId: string, +): Promise<((scheduleFilePath: string) => boolean) | undefined> { + let parentWorkspaceId: string | null | undefined; + let known = false; + try { + const { workspaces } = await wmill.listUserWorkspaces(); + const entry = workspaces?.find((w) => w.id === workspaceId); + known = entry !== undefined; + parentWorkspaceId = entry?.parent_workspace_id; + } catch { + // A fork-scoped token cannot list workspaces. + } + // No parent on record (a fork whose parent was deleted keeps its + // `wm-fork-` id): nothing defers to a parent any more. + if (known && !parentWorkspaceId) { + return undefined; + } + // Without the listing only the `wm-fork-` prefix says fork: a dev + // workspace (custom id) reached with a fork-scoped token counts as none. + if (!isForkWorkspace(workspaceId, parentWorkspaceId)) { + return undefined; + } + let parentPaths: Set | undefined; + if (parentWorkspaceId) { + try { + parentPaths = new Set(); + const perPage = 100; + for (let page = 1; ; page++) { + const batch = await wmill.listSchedules({ + workspace: parentWorkspaceId, + page, + perPage, + }); + batch.forEach((s) => parentPaths!.add(s.path)); + if (batch.length < perPage) break; + } + } catch { + parentPaths = undefined; + } + } + return (scheduleFilePath) => + parentPaths === undefined || + parentPaths.has( + removeType(scheduleFilePath, "schedule").replaceAll(SEP, "/"), + ); +} + export async function push( opts: GlobalOptions & SyncOptions & { repository?: string; branch?: string; + keepDeleted?: boolean; acceptOverridingPermissionedAsWithSelf?: boolean; }, ) { @@ -4574,6 +4824,9 @@ export async function push( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + const parentOwnsScheduleEnabled = opts.includeSchedules + ? await parentOwnedScheduleEnabled(workspace.workspaceId) + : undefined; if (opts.lint) { log.info("Running lint validation before push..."); @@ -4637,6 +4890,10 @@ export async function push( // ignore } + // See ZipFSElement's `localFlowInlineMapping`. + const localFlowInlineMapping = (flowDir: string) => + checkoutInlineNames(path.join(process.cwd(), flowDir, "flow.yaml")); + const remote = ZipFSElement( (await downloadZip( workspace, @@ -4662,6 +4919,7 @@ export async function push( resourceTypeToIsFileset, false, parseSyncBehavior(opts.syncBehavior) >= 1, + localFlowInlineMapping, ); const local = await FSFSElement( @@ -4682,6 +4940,7 @@ export async function push( wsNameForFiles, false, // els1 (local) is not the remote source await isCaseInsensitiveFilesystem(process.cwd()), + parentOwnsScheduleEnabled, ); // Detect resources/variables that the local config flags as ws_specific @@ -4791,6 +5050,13 @@ export async function push( ); } + // After the shared-lock pass, which reads a shared lockfile's deletion as the + // signal that this checkout is not deduplicated — an advisory about the local + // tree that holds whether or not remote items are being kept. + if (opts.keepDeleted) { + dropDeletions(changes, "remote"); + } + const autoRegenerate = !!(opts as any).autoMetadata; const staleScripts: string[] = []; const staleFlows: string[] = []; @@ -4867,6 +5133,14 @@ export async function push( } if (autoRegenerate && tree) { + // Pass 1 only ever walks the change set, so anything imported through a + // module the push leaves alone is still a dead end here. + await resolvePlaceholdersFromLocal( + tree, + localScriptsByRemotePath(localMap), + opts.defaultTs, + ); + // Propagate staleness through imports + upload script content to // raw_script_temp so the dep job can resolve cross-folder relative imports // via temp_script_refs (instead of hitting 404s for not-yet-deployed @@ -5103,7 +5377,10 @@ export async function push( // unchanged (pushSharedUi still runs) and the summary count includes ui/. if (opts.dryRun) { try { - for (const c of await diffSharedUi(workspace.workspaceId)) { + for (const c of await diffSharedUi( + workspace.workspaceId, + opts.keepDeleted, + )) { if (c.type === "added") { changes.push({ name: "added", path: c.path, content: "" }); } else if (c.type === "deleted") { @@ -5275,7 +5552,9 @@ export async function push( : {}), })), total: changes.length, - ...(changes.length > 0 ? { warning: SYNC_PUSH_DESTRUCTIVE_WARNING } : {}), + ...(changes.length > 0 + ? { warning: syncPushDestructiveWarning(opts.keepDeleted) } + : {}), }; console.log(JSON.stringify(result, null, 2)); return; @@ -5339,31 +5618,25 @@ export async function push( if (opts.dryRun) { log.info(colors.gray(`Dry run complete.`)); - log.warn(colors.yellow(`\n⚠ ${SYNC_PUSH_DESTRUCTIVE_WARNING}`)); + log.warn( + colors.yellow(`\n⚠ ${syncPushDestructiveWarning(opts.keepDeleted)}`), + ); return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); @@ -5734,6 +6007,9 @@ export async function push( originalLocalPath: originalWorkspaceSpecificPath, permissionedAsContext, wsSpecific: isWsSpecific ? true : undefined, + enabledOwnedByParent: parentOwnsScheduleEnabled?.( + change.path, + ), keyPushOpts: { noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, @@ -5869,6 +6145,9 @@ export async function push( originalLocalPath: localFilePath, permissionedAsContext, wsSpecific: isAddedWsSpecific ? true : undefined, + enabledOwnedByParent: parentOwnsScheduleEnabled?.( + change.path, + ), keyPushOpts: { noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, @@ -6001,7 +6280,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // Flow folder doesn't exist locally — delete on server @@ -6046,7 +6325,7 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message }, + { message: opts.message, permissionedAsContext }, ); } else { // App folder doesn't exist locally — delete on server @@ -6092,7 +6371,11 @@ export async function push( undefined, opts.plainSecrets ?? false, alreadySynced, - { message: opts.message, defaultTs: opts.defaultTs }, + { + message: opts.message, + defaultTs: opts.defaultTs, + permissionedAsContext, + }, ); } else { // The entire raw app folder was deleted locally, @@ -6313,7 +6596,7 @@ export async function push( } } try { - await pushSharedUi(workspace.workspaceId); + await pushSharedUi(workspace.workspaceId, opts.keepDeleted); } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } @@ -6404,6 +6687,22 @@ export async function push( ), ); } + // Both delete handlers move the item to the workspace trashbin first; without + // this the CLI is the only surface that never says so, and the deletion reads + // as final. + const deletedSecretBearing = changes.filter( + (c) => + c.name === "deleted" && + secretBearingObjectKind(c.path) !== undefined && + !failedChanges.some((f) => f.path === c.path), + ); + if (deletedSecretBearing.length > 0) { + log.info( + colors.gray( + `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it with \`wmill trash list\` and \`wmill trash restore \`, or from Workspace settings -> Trashbin.`, + ), + ); + } if (failedChanges.length > 0) { // Not process.exit: under Node a piped stdout write is async, so exiting // here would truncate the JSON result mid-object for CI consumers. @@ -6415,7 +6714,10 @@ export async function push( let sharedUiPushed = false; if (!opts.dryRun) { try { - sharedUiPushed = await pushSharedUi(workspace.workspaceId); + sharedUiPushed = await pushSharedUi( + workspace.workspaceId, + opts.keepDeleted, + ); } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } @@ -6480,6 +6782,10 @@ const command = new Command() .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") .option("--include-key", "Include workspace encryption key") + .option( + "--keep-deleted", + "Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates.", + ) .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( @@ -6543,6 +6849,10 @@ const command = new Command() "--skip-reencrypt-on-key-change", "When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.", ) + .option( + "--keep-deleted", + "Do not delete remote items that no longer exist locally. Only adds and updates.", + ) .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( diff --git a/cli/src/commands/trash/trash.ts b/cli/src/commands/trash/trash.ts new file mode 100644 index 0000000000..1d571e2884 --- /dev/null +++ b/cli/src/commands/trash/trash.ts @@ -0,0 +1,147 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { apiErrorMessage, formatTimestamp } from "../../utils/utils.ts"; + +async function list( + opts: GlobalOptions & { + json?: boolean; + kind?: string; + page?: number; + limit?: number; + } +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (opts.page !== undefined && opts.page < 1) { + throw new Error("--page starts at 1"); + } + + const items = await wmill.listTrash({ + workspace: workspace.workspaceId, + itemKind: opts.kind, + // The trash endpoint counts pages from 0, unlike the API's other list + // endpoints whose `page` starts at 1; the flag counts from 1 like those. + page: opts.page === undefined ? undefined : opts.page - 1, + perPage: opts.limit, + }); + + if (opts.json) { + console.log(JSON.stringify(items)); + return; + } + if (items.length === 0) { + log.info("No trashed items found."); + return; + } + new Table() + .header(["ID", "Kind", "Path", "Deleted by", "Deleted at", "Expires at"]) + .padding(2) + .border(true) + .body( + items.map((item) => [ + String(item.id), + item.item_kind, + item.item_path, + item.deleted_by, + formatTimestamp(item.deleted_at), + formatTimestamp(item.expires_at), + ]) + ) + .render(); + log.info( + colors.gray( + "`wmill trash get ` shows what an item held, `wmill trash restore ` puts it back." + ) + ); +} + +async function get(opts: GlobalOptions & { json?: boolean }, id: number) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const item = await wmill.getTrashItem({ + workspace: workspace.workspaceId, + id, + }); + + if (opts.json) { + console.log(JSON.stringify(item)); + return; + } + console.log(colors.bold("ID:") + " " + item.id); + console.log(colors.bold("Kind:") + " " + item.item_kind); + console.log(colors.bold("Path:") + " " + item.item_path); + console.log(colors.bold("Deleted by:") + " " + item.deleted_by); + console.log(colors.bold("Deleted at:") + " " + formatTimestamp(item.deleted_at)); + console.log(colors.bold("Expires at:") + " " + formatTimestamp(item.expires_at)); + console.log(colors.bold("Data:")); + console.log(JSON.stringify(item.item_data, null, 2)); +} + +async function restore(opts: GlobalOptions, ...ids: number[]) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + let failed = 0; + for (const id of ids) { + try { + const message = await wmill.restoreTrashItem({ + workspace: workspace.workspaceId, + id, + }); + log.info(colors.green(message)); + } catch (e) { + failed += 1; + log.error( + `Could not restore trash item ${id}: ${apiErrorMessage(e) ?? String(e)}` + ); + } + } + if (failed > 0) { + process.exitCode = 1; + } +} + +const command = new Command() + .description( + "List, inspect and restore items deleted in the last three days (requires admin)" + ) + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("list", "List trashed items, most recently deleted first") + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("get", "Show a trashed item and the data it was deleted with") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any) + .command("restore", "Put trashed items back at their paths") + .arguments("") + .action(restore as any); + +export default command; diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..4e6b0eeb16 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -222,9 +224,12 @@ export async function pushTrigger( } } +// `enabled` is operational state a sync deliberately does not carry: the server strips it from +// the workspace export and the push below never sends it, so a created trigger comes up enabled +// and pausing one stays a local decision. type NativeTriggerFile = Omit< NativeTrigger, - "external_id" | "workspace_id" | "error" + "external_id" | "workspace_id" | "error" | "enabled" >; export async function pushNativeTrigger( @@ -262,6 +267,7 @@ export async function pushNativeTrigger( service_config: result.service_config, error: result.error, summary: result.summary, + enabled: result.enabled, }; log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`); } catch { @@ -620,8 +626,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise; } /** @@ -153,47 +155,52 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + + // The server reads an absent `value_expires_at` as "leave the stored date alone", so + // dropping the key from the file has to be sent as an explicit `null`. `isSuperset` + // only compares keys the file still has, so a removed expiry is also invisible to it: + // without both halves the clear never applies and every later push re-reports it. + const clearsValueExpiresAt = + localVariableBody.value_expires_at === undefined && + (variable as ListableVariable | undefined)?.value_expires_at !== undefined && + (variable as ListableVariable | undefined)?.value_expires_at !== null; + if (variable) { - // The server reads an absent `value_expires_at` as "leave the stored date alone", so - // dropping the key from the file has to be sent as an explicit `null`. `isSuperset` - // only compares keys the file still has, so a removed expiry is also invisible to it: - // without both halves the clear never applies and every later push re-reports it. - const clearsValueExpiresAt = - localVariable.value_expires_at === undefined && - variable.value_expires_at !== undefined && - variable.value_expires_at !== null; - - if (!clearsValueExpiresAt && isSuperset(localVariable, variable)) { + if (!clearsValueExpiresAt && isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - ...(clearsValueExpiresAt ? { value_expires_at: null } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + ...(clearsValueExpiresAt ? { value_expires_at: null } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -201,11 +208,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 219252bb7a..0aca0984ff 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -239,6 +239,7 @@ async function createWorkspaceFork( interface ForkedDatatableInfo { name: string; new_dbname: string; + fork_behavior?: "schema_only" | "schema_and_data"; } const forkedDatatables: ForkedDatatableInfo[] = []; @@ -285,6 +286,23 @@ async function createWorkspaceFork( const newDbName = `${trueWorkspaceId.replace(/-/g, "_")}__${dt.name}`; + const forkBehavior = dtBehavior as "schema_only" | "schema_and_data"; + // A server that reports `permissioned` copies each data table in the fork request itself, + // and refuses a database copied beforehand. An older one only takes a database copied here. + if (typeof dt.permissioned === "boolean") { + log.info( + colors.blue( + ` Datatable "${dt.name}" will be cloned (${forkBehavior === "schema_only" ? "schema" : "schema + data"}) into "${newDbName}" when the fork is created.` + ) + ); + forkedDatatables.push({ + name: dt.name, + new_dbname: newDbName, + fork_behavior: forkBehavior, + }); + continue; + } + try { log.info( colors.blue(` Creating database "${newDbName}" for datatable "${dt.name}"...`) @@ -300,7 +318,7 @@ async function createWorkspaceFork( log.info( colors.blue( - ` Importing ${dtBehavior === "schema_only" ? "schema" : "schema + data"}...` + ` Importing ${forkBehavior === "schema_only" ? "schema" : "schema + data"}...` ) ); @@ -310,7 +328,7 @@ async function createWorkspaceFork( source: `datatable://${dt.name}`, target: `datatable://${dt.name}`, target_dbname_override: newDbName, - fork_behavior: dtBehavior as "schema_only" | "schema_and_data", + fork_behavior: forkBehavior, }, }); @@ -336,6 +354,8 @@ async function createWorkspaceFork( id: trueWorkspaceId, name: opts.createWorkspaceName ?? workspaceName ?? trueWorkspaceId, color: forkColor, + // So a clone the fork would refuse is refused before any branch is created. + forked_datatables: forkedDatatables, }, }); if (gitSyncJobIds && gitSyncJobIds.length > 0) { diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..0c746d2261 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { @@ -630,6 +670,83 @@ export async function getEffectiveSettings( return effective; } +// Resolve workspace name from a --branch override (git branch → workspace name). +// Falls back to using the branch value as-is (backward compat: old key = branch name). +function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string { + const match = findWorkspaceByGitBranch(opts.workspaces, branchName); + return match ? match[0] : branchName; +} + +// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, +// then --workspace → matching config key (incl. when --base-url is set). Returns +// undefined when no flag-based resolution applies; callers then fall back to +// inferWsNameFromProfile on the resolved workspace profile. +export function resolveWsNameForConfigFromFlags( + opts: SyncOptions & { branch?: string; workspace?: string } +): string | undefined { + if (opts.branch) { + return resolveWsNameFromBranch(opts, opts.branch); + } + if (opts.workspace) { + // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, + // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. + const validKeys = getWorkspaceNames(opts.workspaces); + if (validKeys.includes(opts.workspace)) { + return opts.workspace; + } + } + return undefined; +} + +/** + * Match a workspace config entry to a resolved workspace profile by remote + + * workspace id. The fallback for when no flag names the entry outright. + */ +export function inferWsNameFromProfile( + opts: SyncOptions, + profile: { remote: string; workspaceId: string } +): string | undefined { + if (!opts.workspaces) return undefined; + for (const name of getWorkspaceNames(opts.workspaces)) { + const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; + if (!entry?.baseUrl) continue; + try { + const entryUrl = new URL(entry.baseUrl).toString(); + const profileUrl = new URL(profile.remote).toString(); + const entryWsId = entry.workspaceId ?? name; + if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { + return name; + } + } catch { + continue; + } + } + return undefined; +} + +/** + * `syncBehavior` as the workspace being pushed to sees it. The top level alone + * misses a `workspaces..overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 3abd8b1e9a..45eb4e941f 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.798.1"; +export const VERSION = "1.816.0"; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/src/core/log.ts b/cli/src/core/log.ts index e78450597f..c4cf256e9d 100644 --- a/cli/src/core/log.ts +++ b/cli/src/core/log.ts @@ -41,6 +41,12 @@ export function warnStderr(msg: unknown) { console.error(`\x1b[33m${String(msg)}\x1b[39m`); } +// A notice that must reach a log even in silent (`--json-output`) mode: +// stderr keeps stdout parseable, and a job that runs the CLI records both. +export function warnAlways(msg: unknown) { + console.error(`\x1b[33m${String(msg)}\x1b[39m`); +} + export function error(msg: unknown) { console.error(`\x1b[31m${String(msg)}\x1b[39m`); } diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..d0301373da 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,13 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { + extractFolderPath, + isAppFolderMetadataFile, + isRawAppFolderMetadataFile, +} from "../utils/resource_folders.ts"; +import { deploysWithRawApp } from "../utils/app_files.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +17,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map @@ -59,6 +95,45 @@ function contentHasOnBehalfOf(content: string, typeStr: string): boolean { return false; } +type AppTypeStr = "app" | "raw_app"; + +function isAppTypeStr(typeStr: string): typeStr is AppTypeStr { + return typeStr === "app" || typeStr === "raw_app"; +} + +/** The app folder a file belongs to. `isAppFolderMetadataFile` and its raw twin + * match a literal `/`, unlike `extractFolderPath` — so normalize before either, + * or a Windows path takes a different branch from the same file on Linux. */ +function appFolderOf(path: string, typeStr: AppTypeStr): string { + return extractFolderPath(path, typeStr) ?? path; +} + +function toPosix(path: string): string { + return path.replaceAll("\\", "/"); +} + +/** App folders whose own metadata file is being added or deleted, which is how a + * whole app arrives or goes rather than being redeployed. Neither takes an owner + * over: a create has none yet, and a delete leaves none behind. */ +function appsArrivingOrLeaving(changes: Change[]): Set { + const folders = new Set(); + for (const change of changes) { + if (change.name === "edited") continue; + const path = toPosix(change.path); + if (!isAppFolderMetadataFile(path) && !isRawAppFolderMetadataFile(path)) { + continue; + } + let typeStr: string; + try { + typeStr = getTypeStrFromPath(path); + } catch { + continue; + } + if (isAppTypeStr(typeStr)) folders.add(appFolderOf(path, typeStr)); + } + return folders; +} + export async function preCheckPermissionedAs( changes: Change[], userEmail: string, @@ -71,6 +146,12 @@ export async function preCheckPermissionedAs( if (userIsAdminOrDeployer) return; const wouldChangeItems: { path: string; currentOwner: string }[] = []; + const addItem = (item: { path: string; currentOwner: string }) => { + if (!wouldChangeItems.some((i) => i.path === item.path)) { + wouldChangeItems.push(item); + } + }; + const arrivingOrLeaving = appsArrivingOrLeaving(changes); for (const change of changes) { let typeStr: string; @@ -80,6 +161,22 @@ export async function preCheckPermissionedAs( continue; } + // An app is redeployed whole by any change to any of the files it actually + // sends — added, edited or deleted alike — so its policy is rewritten + // regardless of what the file holds. Settled here, before the content the + // other kinds parse to find their owner, which an app has none of to parse. + if (isAppTypeStr(typeStr)) { + const path = toPosix(change.path); + const folder = appFolderOf(path, typeStr); + if ( + !arrivingOrLeaving.has(folder) && + (typeStr === "app" || deploysWithRawApp(path.slice(folder.length))) + ) { + addItem({ path: folder, currentOwner: "(app policy owner)" }); + } + continue; + } + if (change.name === "added") { const content = change.content; if (!content) continue; @@ -100,11 +197,6 @@ export async function preCheckPermissionedAs( const label = typeStr === "script" ? "(script owner)" : "(flow owner)"; wouldChangeItems.push({ path: change.path, currentOwner: label }); - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); } continue; } @@ -147,12 +239,6 @@ export async function preCheckPermissionedAs( } } continue; - } else if (typeStr === "app") { - wouldChangeItems.push({ - path: change.path, - currentOwner: "(app policy owner)", - }); - continue; } else if (typeStr === "schedule") { const match = beforeContent.match( /email:\s*["']?([^\s"']+)["']?/ diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 552c57cf6b..e3ab2d4bc2 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -237,9 +237,15 @@ export async function pushWorkspaceSettings( } // Exclude fields that are never applied here: slack_team_id/slack_name are OAuth-only, - // and name is not applied on pull (see below), so a name-only diff stays a no-op. + // and name is never applied (see below), so a name-only diff stays a no-op. color is + // applied only when the file carries it, so an unset one leaves the comparison too. const { slack_team_id: _lst, slack_name: _lsn, name: _ln, ...comparableLocal } = localSettings; const { slack_team_id: _rst, slack_name: _rsn, name: _rn, ...comparableRemote } = settings; + const colorManaged = localSettings.color != null; + if (!colorManaged) { + delete comparableLocal.color; + delete comparableRemote.color; + } if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; @@ -256,8 +262,19 @@ export async function pushWorkspaceSettings( }); } - // Handle auto_invite using grouped format - if (!deepEqual(localSettings.auto_invite, settings.auto_invite)) { + // Handle auto_invite using grouped format. The domain invite and the instance groups + // are applied by separate endpoints, each rewriting only its own keys. + const { + instance_groups: localGroups, + instance_groups_roles: localGroupRoles, + ...localDomainInvite + } = localSettings.auto_invite ?? {}; + const { + instance_groups: remoteGroups, + instance_groups_roles: remoteGroupRoles, + ...remoteDomainInvite + } = settings.auto_invite ?? {}; + if (!deepEqual(localDomainInvite, remoteDomainInvite)) { log.debug(`Updating auto invite...`); const localAutoInvite = localSettings.auto_invite; @@ -295,6 +312,20 @@ export async function pushWorkspaceSettings( } } + // Only when settings.yaml declares instance_groups: clearing a group removes the + // workspace members it granted, so an absent key must never clear it. + if ( + localGroups != undefined && + (!deepEqual(localGroups, remoteGroups) || + !deepEqual(localGroupRoles ?? {}, remoteGroupRoles ?? {})) + ) { + log.debug(`Updating instance groups...`); + await wmill.editInstanceGroups({ + workspace, + requestBody: { groups: localGroups, roles: localGroupRoles ?? {} }, + }); + } + if (!deepEqual(localSettings.ai_config, settings.ai_config)) { log.debug(`Updating copilot settings...`); await wmill.editCopilotConfig({ @@ -394,10 +425,9 @@ export async function pushWorkspaceSettings( }); } - // Workspace display name is intentionally not applied on pull: settings.yaml is shared - // across a repo's branches, so applying it would let one workspace's name overwrite - // another's when both sync the same repo. It stays in the file (written on push), but a - // live workspace is only renamed by its owner. + // Workspace display name is intentionally never applied by `sync push`: settings.yaml is + // shared across a repo's branches, so applying it would let one workspace's name overwrite + // another's when both sync the same repo. `sync pull` still records it. if (localSettings.mute_critical_alerts != settings.mute_critical_alerts) { log.debug(`Updating mute critical alerts...`); @@ -409,7 +439,9 @@ export async function pushWorkspaceSettings( }); } - if (localSettings.color != settings.color) { + // A color is applied only when the file carries one: `sync pull` omits the key for a + // workspace without a color, so an unset key means "not managed by git", never "clear". + if (colorManaged && localSettings.color != settings.color) { log.debug(`Updating workspace color...`); await wmill.changeWorkspaceColor({ workspace, @@ -429,10 +461,17 @@ export async function pushWorkspaceSettings( if (!deepEqual(localSettings.datatable, settings.datatable)) { log.debug(`Updating datatable config...`); - await wmill.editDataTableConfig({ + const { stranded_references } = await wmill.editDataTableConfig({ workspace, requestBody: { settings: localSettings.datatable ?? { datatables: {} } }, }); + if (stranded_references?.length) { + log.warn( + `Removed data tables governed data tables in other workspaces, which no longer resolve: ${stranded_references + .map((r) => `${r.workspace_id}/${r.datatable}`) + .join(", ")}. A superadmin can point them somewhere else.`, + ); + } } if (localSettings.slack_command_script != settings.slack_command_script) { diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 100bea6e53..202a6feaad 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -115,6 +115,8 @@ Local previews exist for every entity type and don't deploy: - \`wmill flow preview -d ''\` — run a local flow.yaml. - \`wmill app dev\` — live-reload dev server for raw apps. +An argument typed as a resource takes the bare string \`"$res:"\` as its whole value (a variable takes \`"$var:"\`) — never an object wrapper like \`{"$res": ""}\`, and never a plain path. See the \`resources\` skill. + Argument shapes and per-language details live in the \`write-script-\`, \`write-flow\`, and \`raw-app\` skills. ## Keeping metadata in sync @@ -163,6 +165,8 @@ No CI workflow runs \`wmill sync push\` automatically, so deploy directly from t - \`wmill sync push --dry-run\` to preview. - \`wmill sync push\` to apply. +A push deletes remote items that have no local file. They land in the workspace trashbin for three days: \`wmill trash list\` shows them and \`wmill trash restore \` puts one back (both need a workspace admin). + ### In both cases Only deploy when the user explicitly asks to deploy, publish, push, or ship — not when they say "run", "try", or "test". For testing local edits use the per-entity \`preview\` commands (\`wmill script preview\`, \`wmill flow preview\`) — they don't deploy. diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index b5cb3417f4..81895997b9 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1037,12 +1037,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -1106,6 +1110,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord /** * Create a SQL template function for PostgreSQL/datatable queries * @param name - Database/datatable name (default: "main") + * @param opts.role - Connect as this data table role instead of the data table's default one. + * Only meaningful on a data table under roles, and only for a role you are a tenant of. * @returns SQL template function for building parameterized queries * @example * let sql = wmill.datatable() @@ -1115,8 +1121,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * SELECT * FROM friends * WHERE name = \${name} AND age = \${age}::int * \`.fetch() + * @example + * // Read through a restricted role + * let sql = wmill.datatable("main", { role: "analytics" }) */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -1824,12 +1833,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -1893,6 +1906,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord /** * Create a SQL template function for PostgreSQL/datatable queries * @param name - Database/datatable name (default: "main") + * @param opts.role - Connect as this data table role instead of the data table's default one. + * Only meaningful on a data table under roles, and only for a role you are a tenant of. * @returns SQL template function for building parameterized queries * @example * let sql = wmill.datatable() @@ -1902,8 +1917,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * SELECT * FROM friends * WHERE name = \${name} AND age = \${age}::int * \`.fetch() + * @example + * // Read through a restricted role + * let sql = wmill.datatable("main", { role: "analytics" }) */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -2705,12 +2723,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2774,6 +2796,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord /** * Create a SQL template function for PostgreSQL/datatable queries * @param name - Database/datatable name (default: "main") + * @param opts.role - Connect as this data table role instead of the data table's default one. + * Only meaningful on a data table under roles, and only for a role you are a tenant of. * @returns SQL template function for building parameterized queries * @example * let sql = wmill.datatable() @@ -2783,8 +2807,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * SELECT * FROM friends * WHERE name = \${name} AND age = \${age}::int * \`.fetch() + * @example + * // Read through a restricted role + * let sql = wmill.datatable("main", { role: "analytics" }) */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -4393,10 +4420,13 @@ def send_teams_message(conversation_id: str, text: str, success: bool = True, ca # # Args: # name: Database name (default: "main") +# role: Connect as this data table role instead of the data table's default one. +# Only meaningful on a data table under roles, and only for a role you are a +# tenant of. # # Returns: # DataTableClient instance -def datatable(name: str = 'main') +def datatable(name: str = 'main', *, role: Optional[str] = None) # Get a DuckLake client for DuckDB queries. # @@ -4580,6 +4610,30 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -4587,10 +4641,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -4598,10 +4657,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -4609,7 +4670,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -4650,13 +4711,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -4670,7 +4735,7 @@ async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_ # ... # # results = await parallel(items, process, concurrency=5) -async def parallel(items, fn, concurrency: Optional[int] = None) +async def parallel(items, fn, *, concurrency: Optional[int] = None) # Commit Kafka offsets for a trigger with auto_commit disabled. # @@ -5116,6 +5181,8 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute \`wmill flow preview -d ''\` directly — pick plausible args from the flow's input schema. +An input typed as a resource (\`format: resource-\` in the schema) takes the bare string \`"$res:"\` as its whole value — \`-d '{"db": "$res:f/databases/postgres_prod"}'\`, not \`{"db": {"$res": "..."}}\` and not a plain path. Same for a variable, with \`"$var:"\`. See the \`resources\` skill. + \`wmill flow preview\` is safe to run yourself (it does not deploy). \`wmill generate-metadata\` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's \`AGENTS.md\` opts into automatic metadata. After running it, check the regenerated \`.lock\` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only \`wmill sync push\` deploys; run it only when the user explicitly asks. ### Visual preview @@ -5227,9 +5294,60 @@ tool, \`websearch\` for web search. } \`\`\` -- \`provider\` is a static object, not a bare resource string: \`{ "kind": , +- \`provider\` is an object, not a bare resource string: \`{ "kind": , "resource": "$res:", "model": }\`. Required unless the module links to a saved - agent through \`value.agent\` + agent through \`value.agent\`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below + +### Chat-Mode Flows + +A flow with \`value.chat_input_enabled: true\` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required \`user_message\` string +input, read by the agent. Any other flow input the composer does not edit itself is asked for +under Configure inputs. + +**A static \`provider\` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (\`"expr": "flow_input.model_config"\`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +control — and a field left static is fixed, with no control drawn for it. \`kind\` is the one +exception: the composer writes it only together with \`resource\`, since a provider is picked as a +pair, so a \`kind\` input wired on its own stays askable under Configure inputs and nothing the run +needs becomes unreachable. + +\`\`\`json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "window", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +\`\`\` + +- Wiring field by field means one object literal whose values are literals or bare \`flow_input.x\` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare \`flow_input.x\` for the whole object + is read instead as that one input carrying every field +- \`memory\` is what lets the agent see earlier turns; without it every message starts from nothing +- \`streaming\` on makes the answer and its thinking appear token by token instead of all at once +- \`user_attachments\` points at a flow input typed as an array of s3 objects + (\`{ "type": "array", "items": { "type": "object", "resourceType": "s3object" } }\`), so files + sent with a message reach the agent +- Running one needs a \`memory_id\` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job ### Tool Naming Rules @@ -5533,7 +5651,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\`: it is ignored when memory is off, and an older \`auto\` or\\n\`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -5708,6 +5826,8 @@ data: tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema + roles: # Optional: the role the app uses each datatable through + main: analyst \`\`\` **Table reference formats:** @@ -5715,6 +5835,8 @@ data: - \`/
\` — Specific table in public schema - \`/:
\` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The \`sql_to_apply/\` folder is for creating/modifying database tables during development. @@ -5866,6 +5988,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. + +\`\`\`tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +\`\`\` + +\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -5958,8 +6093,8 @@ export async function main(user_id: string) { const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`.execute(); + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`.execute(); return user; } @@ -5978,8 +6113,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user \`\`\` @@ -5988,12 +6123,14 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — \`get_user\`, not \`a\`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. -6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +3. **Terminate every datatable statement** — the tagged template and \`db.query(...)\` only build a statement. It runs when you call \`fetch\` / \`fetchOne\` / \`fetchOneScalar\` / \`execute\` (\`fetch\` / \`fetch_one\` / \`fetch_one_scalar\` / \`execute\` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — \`get_user\`, not \`a\`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. +7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers @@ -6188,6 +6325,41 @@ Reference other resources: } \`\`\` +## Passing a Resource or Variable as a Run Argument + +A script or flow argument typed as a resource (schema \`format: resource-\`) is passed as +the **bare string** \`$res:\` — the whole argument value. Same for a variable, with +\`$var:\`. This applies everywhere job arguments are supplied: \`wmill script run/preview\`, +\`wmill flow run/preview\`, the \`runScriptByPath\` / \`runFlowByPath\` API, a schedule's \`args\`, a +trigger's configured static args. + +\`\`\`json +{ + "db": "$res:f/databases/postgres_prod", + "api_token": "$var:g/all/api_token" +} +\`\`\` + +The reference is resolved when the job runs, under the job's run-as identity — the caller for an +ordinary run, but the configured principal for a schedule, a trigger, or a runnable set to run on +behalf of someone else. The run fails if that identity cannot read the referenced resource or +variable. + +**Never wrap it in an object.** The resolver only rewrites a JSON value that *is* a string +starting with \`$res:\` / \`$var:\`; keys are never inspected. These are all wrong and are passed +through to the script unchanged: + +\`\`\`json +{ "db": { "$res": "f/databases/postgres_prod" } } +{ "db": { "resource": "f/databases/postgres_prod" } } +{ "db": "f/databases/postgres_prod" } +\`\`\` + +The string may sit anywhere a string can — a top-level argument, a nested object field +(\`{ "gh_auth": { "token": "$var:g/all/gh_token" } }\`), or an array element (array elements are +walked only while nested at most two levels deep, and only for arrays of at most 1000 items). +The prefix must be on the string itself. + ## Common Resource Types ### PostgreSQL @@ -6620,14 +6792,50 @@ A caught failure reads the same whether it came from a task or from a \`step()\` Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` \`\`\`typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and \`attempts\`. Retries with no + * \`delay\` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: \`2\` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A \`taskScript\` + * target is keyed on the arguments it is called with. It has no effect on a + * \`taskFlow\` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -6645,9 +6853,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -6703,12 +6913,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -6791,6 +7005,30 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -6798,10 +7036,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -6809,10 +7052,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -6820,7 +7065,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -6861,13 +7106,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. # @@ -7074,11 +7323,11 @@ flow related commands - \`flow push \` - push a local flow spec. This overrides any remote versions. - \`--message \` - Deployment message - \`flow run \` - run a flow by path. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag). - \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. Pass --step to run only one module in isolation (resolves nested steps inside branchone/branchall/forloopflow/whileloopflow plus the special preprocessor/failure modules; supported step types: rawscript, script, flow). - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`--step \` - Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does. @@ -7289,7 +7538,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory +Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file **Arguments:** \`[directory:string]\` @@ -7474,11 +7723,11 @@ script related commands - \`--json\` - Output as JSON (for piping to jq) - \`script show \` - show a script's content (alias for get) - \`script run \` - run a script by path - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. - \`--tag \` - Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script preview \` - preview a local script without deploying it. Supports both regular and codebase scripts. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. A resource argument is the bare string $res: as its whole value, and a variable argument is the bare string $var: — not an object wrapper keyed on $res/$var, and not a plain path. - \`-s --silent\` - Do not output anything other than the final output. Useful for scripting. - \`--tag \` - Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the script's default tag). - \`script new \` - create a new script @@ -7519,6 +7768,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key + - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes @@ -7550,6 +7800,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. + - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) @@ -7580,6 +7831,27 @@ Manage API tokens - \`--expiration \` - Token expiration (ISO 8601 timestamp) - \`token delete \` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) +- \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- \`--limit \` - Number of items to return (default 100, max 1000) +- \`--page \` - Page to return, starting at 1 + +**Subcommands:** + +- \`trash list\` - List trashed items, most recently deleted first + - \`--json\` - Output as JSON (for piping to jq) + - \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - \`--limit \` - Number of items to return (default 100, max 1000) + - \`--page \` - Page to return, starting at 1 +- \`trash get \` - Show a trashed item and the data it was deleted with + - \`--json\` - Output as JSON (for piping to jq) +- \`trash restore \` - Put trashed items back at their paths + ### trigger trigger related commands @@ -7935,7 +8207,9 @@ properties: at once (1-65535) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -8383,9 +8657,26 @@ properties: type: boolean description: If true, passes the request body as a raw string instead of parsing as JSON + allowed_origins: + type: array + items: + type: string + description: 'Origins allowed to call this route cross-origin, matched against + the request''s Origin header (ignoring case) and echoed back on a match. When + set, the list governs both the preflight and the response, overriding any Access-Control-Allow-Origin + the runnable returns via wm_headers. Use [''*''] to opt out of any restriction, + including the http_route_default_allowed_origins instance setting. An empty + list is not a configuration and resolves exactly as null does. When null, the + instance setting applies, or Access-Control-Allow-Origin: * if it is unset. + Ignored on a static website, which has no authentication of its own and so hands + out public files: restricting which browsers may read them protects nothing + while breaking cross-origin webfonts and fetches. A single-file static asset + is not exempt, since it can carry an authentication_method.' error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -8552,7 +8843,9 @@ properties: endpoint. error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -8676,7 +8969,9 @@ properties: - v5 error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -8772,7 +9067,9 @@ properties: description: Array of NATS subjects to subscribe to error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -8862,7 +9159,9 @@ properties: description: Name of the PostgreSQL logical replication slot to use error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -9109,7 +9408,9 @@ properties: description: Array of SQS message attribute names to include with each message error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow @@ -9308,7 +9609,9 @@ properties: The extracted value replaces {{state}} in the heartbeat message. error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare path, + without the script/ or flow/ prefix a schedule error handler takes; it cannot + be a flow. error_handler_args: type: object description: The arguments to pass to the script or flow diff --git a/cli/src/main.ts b/cli/src/main.ts index ccce20da30..fa894b6f2d 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -29,7 +29,7 @@ import lint from "./commands/lint/lint.ts"; import dev from "./commands/dev/dev.ts"; import { GlobalOptions } from "./types.ts"; import { OpenAPI } from "../gen/index.ts"; -import { getHeaders } from "./utils/utils.ts"; +import { apiErrorMessage, getHeaders } from "./utils/utils.ts"; import { detectAuthGatewayChallenge } from "./utils/http_guards.ts"; import { setShowDiffs } from "./core/conf.ts"; import { markRequestsAsCliClient } from "./core/client.ts"; @@ -48,6 +48,7 @@ import job from "./commands/job/job.ts"; import group from "./commands/group/group.ts"; import audit from "./commands/audit/audit.ts"; import token from "./commands/token/token.ts"; +import trash from "./commands/trash/trash.ts"; import generateMetadata from "./commands/generate-metadata/generate-metadata.ts"; import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; @@ -214,6 +215,7 @@ const command = new Command() .command("group", group) .command("audit", audit) .command("token", token) + .command("trash", trash) .command("generate-metadata", generateMetadata) .command("docs", docs) .command("config", config) @@ -321,14 +323,9 @@ async function main() { await command.parse(args); } catch (e) { - if (e && typeof e === "object" && "name" in e && e.name === "ApiError") { - const body = (e as any).body; - let bodyStr = typeof body === "object" && body !== null ? JSON.stringify(body) : String(body ?? ""); - // Strip backend source file references like (flows.rs:1400) or @scripts.rs:123:45 - bodyStr = bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); - log.error( - "Server failed. " + (e as any).statusText + ": " + bodyStr - ); + const apiError = apiErrorMessage(e); + if (apiError !== undefined) { + log.error("Server failed. " + apiError); } else if (e instanceof Error) { log.error(e.message); } else if (e !== undefined && e !== null) { diff --git a/cli/src/types.ts b/cli/src/types.ts index c9244a8f4c..5713d06390 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -189,6 +189,8 @@ export interface PushObjOptions { keyPushOpts?: PushWorkspaceKeyOptions; /** TypeScript runtime a bare `.ts` denotes, for raw-app runnables */ defaultTs?: "bun" | "deno"; + /** schedule push into a fork: the file's `enabled` is the parent's */ + enabledOwnedByParent?: boolean; } /** @@ -217,6 +219,7 @@ export async function pushObj( wsSpecific, keyPushOpts, defaultTs, + enabledOwnedByParent, } = opts; const typeEnding = getTypeStrFromPath(p); @@ -231,7 +234,7 @@ export async function pushObj( if (!rawAppName) { throw new Error(`Could not extract raw app name from path: ${p}`); } - await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs); + await pushRawApp(workspace, rawAppName, buildFolderPath(rawAppName, "raw_app"), message, defaultTs, permissionedAsContext); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { @@ -250,7 +253,7 @@ export async function pushObj( } else if (typeEnding === "resource-type") { await pushResourceType(workspace, p, befObj, newObj); } else if (typeEnding === "schedule") { - await pushSchedule(workspace, p, befObj, newObj, permissionedAsContext); + await pushSchedule(workspace, p, befObj, newObj, permissionedAsContext, enabledOwnedByParent); } else if (typeEnding === "http_trigger") { await pushTrigger("http", workspace, p, befObj, newObj, permissionedAsContext); } else if (typeEnding === "websocket_trigger") { diff --git a/cli/src/utils/app_files.ts b/cli/src/utils/app_files.ts new file mode 100644 index 0000000000..ca1be1de0e --- /dev/null +++ b/cli/src/utils/app_files.ts @@ -0,0 +1,48 @@ +import { + APP_BACKEND_FOLDER, + RECORDINGS_FOLDER, +} from "../commands/app/app_metadata.ts"; + +/** Directories under a raw app that no push sends. */ +const NEVER_DEPLOYED_DIRS = new Set([ + "node_modules", + "dist", + ".claude", + "sql_to_apply", +]); + +/** Files under a raw app that no push sends. */ +const NEVER_DEPLOYED_FILES = new Set([ + "package-lock.json", + "DATATABLES.md", + "AGENTS.md", + "wmill.d.ts", +]); + +/** + * Whether an app-root-relative path (`/` separators, leading slash optional) + * reaches the server through any of a push's three channels: `raw_app.yaml` as + * metadata, the backend folder as runnables, the rest bundled by + * `collectAppFiles`. A path this rejects deploys nothing, so changing it is not + * a change to the app however much the sync diff lists it. `collectAppFiles` + * must not drift from this — it reads the same two sets. + */ +export function deploysWithRawApp(relativePath: string): boolean { + const segments = relativePath.split("/").filter(Boolean); + if (segments.length === 0) return false; + const name = segments[segments.length - 1]; + const dirs = segments.slice(0, -1); + // The sets below describe the bundle, which never walks into the backend + // folder — applying them there would strip a runnable whose file shares a + // name (`backend/wmill.d.ts` is the runnable `wmill.d`). Depth 1 because + // `loadRunnablesFromBackend` reads that folder's top level only. + if (dirs[0] === APP_BACKEND_FOLDER) return dirs.length === 1; + if (NEVER_DEPLOYED_FILES.has(name)) return false; + if (dirs.some((d) => NEVER_DEPLOYED_DIRS.has(d))) return false; + // Session recordings are written at the app root only, so an app with a + // `recordings/` component folder of its own still ships it. + if (dirs[0] === RECORDINGS_FOLDER) return false; + return true; +} + +export { NEVER_DEPLOYED_DIRS, NEVER_DEPLOYED_FILES }; diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts index b0b75e659e..959a77b478 100644 --- a/cli/src/utils/dependency_tree.ts +++ b/cli/src/utils/dependency_tree.ts @@ -5,7 +5,7 @@ import { Workspace } from "../commands/workspace/workspace.ts"; import * as wmill from "../../gen/services.gen.ts"; import type { ScriptLang } from "../../gen/types.gen.ts"; -import { ScriptLanguage } from "./script_common.ts"; +import { ScriptLanguage, inferContentTypeFromFilePath } from "./script_common.ts"; import { filterWorkspaceDependencies, generateScriptHash, @@ -14,6 +14,65 @@ import { updateMetadataGlobalLock, } from "./metadata.ts"; import { generateHash } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; + +/** A local script file, keyed in `LocalScripts` by its Windmill remote path. */ +export interface LocalScriptSource { + localPath: string; + content: string; +} +export type LocalScripts = Map; + +/** + * Give every import target that is only a placeholder its local content and its + * own imports, so the graph continues through it. A tree seeded from a subset of + * the checkout otherwise dead-ends at any module outside that subset — a + * re-export barrel needing no edit, typically — hiding what it re-exports from + * `getTempScriptRefs`, which then resolves it against the deployed copy. + */ +export async function resolvePlaceholdersFromLocal( + tree: DoubleLinkedDependencyTree, + localScripts: LocalScripts, + defaultTs: "bun" | "deno" | undefined +): Promise { + // A module resolved here can expose placeholders of its own (a barrel behind + // a barrel), so keep going until a round resolves nothing. + for (;;) { + let resolved = false; + for (const remotePath of tree.placeholderPaths()) { + const local = localScripts.get(remotePath); + if (!local) continue; + let language: ScriptLanguage; + try { + language = inferContentTypeFromFilePath(local.localPath, defaultTs); + } catch { + // A bare `.sql` names no dialect, so its imports cannot be read here. + continue; + } + const imports = await extractRelativeImports( + local.content, + remotePath, + language + ); + // Never directly stale: it is outside the change set, so nothing relocks + // it. It is here to carry edges, and to be uploaded if it differs from + // what is deployed. + await tree.addNode( + remotePath, + local.content, + language, + "", + imports, + "script", + remotePath, + local.localPath, + false + ); + resolved = true; + } + if (!resolved) break; + } +} /** * Diff local scripts against deployed versions, upload only those that differ. @@ -97,6 +156,9 @@ interface DependencyNode { originalPath: string; // Original path passed to handler (with extension for scripts) isRawApp?: boolean; // Only set for apps isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) + // True while the node exists only because something imports it, so it carries + // no content and no imports of its own. + isPlaceholder: boolean; } export class DoubleLinkedDependencyTree { @@ -130,9 +192,11 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } const node = this.nodes.get(path)!; + node.isPlaceholder = false; node.content = content; node.stalenessHash = stalenessHash; node.language = language; @@ -155,7 +219,7 @@ export class DoubleLinkedDependencyTree { stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "dependencies", folder: "", originalPath: depsPath, - isDirectlyStale: !isUpToDate, + isDirectlyStale: !isUpToDate, isPlaceholder: false, }); } } @@ -169,6 +233,7 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } this.nodes.get(importPath)!.importedBy.add(path); @@ -309,6 +374,18 @@ export class DoubleLinkedDependencyTree { return this.nodes.keys(); } + /** + * Paths that exist only as somebody's import target, so the traversal stops + * at them instead of continuing into what they themselves import. + */ + placeholderPaths(): string[] { + const result: string[] = []; + for (const [path, node] of this.nodes.entries()) { + if (node.isPlaceholder) result.push(path); + } + return result; + } + /** * Returns paths of all stale nodes (those with a staleReason). */ diff --git a/cli/src/utils/esbuild_loader.ts b/cli/src/utils/esbuild_loader.ts index 4801c51a46..83d44f4534 100644 --- a/cli/src/utils/esbuild_loader.ts +++ b/cli/src/utils/esbuild_loader.ts @@ -25,7 +25,7 @@ type Esbuild = typeof import("esbuild"); // Version to fall back to if the native host's version can't be read. Keep in // sync with the "esbuild" pin in cli/package.json. -const FALLBACK_VERSION = "0.28.0"; +const FALLBACK_VERSION = "0.28.2"; let cached: Esbuild | undefined; let inFlight: Promise | undefined; diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index 5ca06f427a..354e15c36e 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -1,6 +1,7 @@ import * as log from "../core/log.ts"; import { execSync, spawnSync } from "node:child_process"; import { WM_FORK_PREFIX } from "../core/constants.ts"; +import { SHARED_LOCK_DIR } from "./script_common.ts"; // Fork *workspace id* prefix ("wm-fork-"). WM_FORK_PREFIX is the *branch* // prefix ("wm-fork") used inside the wm-fork// branch name. @@ -584,6 +585,11 @@ export function gitSyncDeployPush(params: { git(["add", "wmill-lock.yaml", `${parent_path}**`], { allowFail: true }); } } + // A shared lockfile (`dedupeLockfiles`) lives under `locks/`, outside every + // item's path glob, and the pull rewrites it when a deployed script's lock + // changed. `-A` also stages the deletion of a swept one; the add fails only + // when nothing under `locks/` exists or is tracked. + git(["add", "-A", "--", SHARED_LOCK_DIR], { allowFail: true }); // `git diff --cached --quiet` exits 1 iff there is something staged. const staged = git(["diff", "--cached", "--quiet"], { allowFail: true }); diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 489a877af7..0801c03911 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -353,6 +353,23 @@ export function formatTimestamp(ts: string): string { return new Date(ts).toISOString().replace("T", " ").substring(0, 19); } +/** + * ": " for an error thrown by the generated API client, + * undefined for anything else. Backend source references such as + * `(flows.rs:1400)` are stripped from the body. + */ +export function apiErrorMessage(e: unknown): string | undefined { + if (!(e && typeof e === "object" && "name" in e && e.name === "ApiError")) { + return undefined; + } + const { body, statusText } = e as { body?: unknown; statusText?: string }; + const bodyStr = + typeof body === "object" && body !== null + ? JSON.stringify(body) + : String(body ?? ""); + return statusText + ": " + bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); +} + /** * Validate that required arguments are present when no -d data was provided. * Fetches the schema from the API and checks required fields. diff --git a/cli/test/app_access_mode_unit.test.ts b/cli/test/app_access_mode_unit.test.ts new file mode 100644 index 0000000000..42a3fd082a --- /dev/null +++ b/cli/test/app_access_mode_unit.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { + executionModeForPush, + generatingPolicy, + markAccessFromPolicy, +} from "../src/commands/app/app.ts"; + +// The access mode is the one policy field a tracked app keeps; a pull then a push must +// deploy the mode that was pulled, guest included, not a default. +test("the access mode survives the app.yaml round trip", async () => { + const guest: any = { policy: { execution_mode: "guest" }, value: {} }; + markAccessFromPolicy(guest); + guest.policy = undefined; + expect(guest.guests).toBe(true); + expect(guest.public).toBeUndefined(); + expect(executionModeForPush(guest, undefined)).toBe("guest"); + await generatingPolicy( + guest, + "u/test/app", + executionModeForPush(guest, undefined), + undefined, + ); + expect(guest.policy.execution_mode).toBe("guest"); + + const anonymous: any = { policy: { execution_mode: "anonymous" }, value: {} }; + markAccessFromPolicy(anonymous); + anonymous.policy = undefined; + expect(anonymous.public).toBe(true); + expect(executionModeForPush(anonymous, undefined)).toBe("anonymous"); + + expect(executionModeForPush({ policy: { execution_mode: "publisher" } }, undefined)).toBe("publisher"); + expect(executionModeForPush({}, undefined)).toBe("publisher"); +}); + +// `viewer` is the narrowest mode — each runnable runs as the viewer, not as the +// app's identity — and the only one with no marker in the file, so both ways it +// can reach a push must survive rather than widen to `publisher`. +test("viewer is never widened to publisher by a push", () => { + // Carried over from the deployed app: a pull writes no marker for it. + expect(executionModeForPush({}, { execution_mode: "viewer" })).toBe("viewer"); + // Stated by the file, which is all a first push has to go on. + expect(executionModeForPush({ policy: { execution_mode: "viewer" } }, undefined)).toBe("viewer"); + // The open-access markers still win, in either direction. + expect(executionModeForPush({ public: true }, { execution_mode: "viewer" })).toBe("anonymous"); + expect(executionModeForPush({}, { execution_mode: "anonymous" })).toBe("publisher"); + // A stated mode is authoritative both ways: the carry-over is for a file that + // says nothing, so it must not pin a deployed app to `viewer` forever. + expect( + executionModeForPush({ policy: { execution_mode: "publisher" } }, { execution_mode: "viewer" }) + ).toBe("publisher"); +}); diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); diff --git a/cli/test/dbt_optional_descriptor_unit.test.ts b/cli/test/dbt_optional_descriptor_unit.test.ts index f44bdc6103..685dc20e8f 100644 --- a/cli/test/dbt_optional_descriptor_unit.test.ts +++ b/cli/test/dbt_optional_descriptor_unit.test.ts @@ -7,7 +7,11 @@ import { expect, test, describe, beforeEach, afterEach } from "bun:test"; import * as fs from "node:fs"; import * as path from "node:path"; import * as os from "node:os"; -import { FSFSElement, elementsToMap } from "../src/commands/sync/sync.ts"; +import { + FSFSElement, + dropDeletions, + elementsToMap, +} from "../src/commands/sync/sync.ts"; import { listWorkspacePaths } from "../src/commands/dev/dev.ts"; import { DbtPathCollisionError, @@ -293,6 +297,47 @@ describe("a dbt project without a descriptor", () => { }); }); +// Unguarded, `--keep-deleted` lets a pull remove the very file it promises to +// keep — the descriptor's warehouse and run arguments with it. +describe("--keep-deleted and a descriptor the remote no longer names", () => { + const DESCRIPTOR = "f/analytics/analytics__dbt/wm_dbt.yaml"; + const emptied = { + name: "edited", + path: DESCRIPTOR, + before: "warehouse: wh\n", + after: "", + }; + const unknownToTheMap = { name: "added", path: DESCRIPTOR, content: "" }; + const changeset = (descriptor: unknown) => + [ + descriptor, + { name: "edited", path: "f/other/script.ts", before: "a", after: "b" }, + ] as Parameters[0]; + + test("survives a pull", () => { + const changes = changeset(emptied); + dropDeletions(changes, "local"); + expect(changes.map((c) => c.path)).toEqual(["f/other/script.ts"]); + }); + + // A stateful pull compares `.wmill`, so a descriptor missing from the compared + // map can still be a working-tree file holding real settings. + test("survives a pull that compared a map without it", () => { + const changes = changeset(unknownToTheMap); + dropDeletions(changes, "local"); + expect(changes.map((c) => c.path)).toEqual(["f/other/script.ts"]); + }); + + test("is still pushed, deleting no remote item", () => { + const changes = changeset(emptied); + dropDeletions(changes, "remote"); + expect(changes.map((c) => c.path)).toEqual([ + DESCRIPTOR, + "f/other/script.ts", + ]); + }); +}); + // `wmill dev` walks basenames, and a dbt script is a DIRECTORY whose descriptor // may not exist — so it is recognized by the project folder or not at all. The // walk must also stop there: the project's own `.sql` models match the script diff --git a/cli/test/dependency_tree_unit.test.ts b/cli/test/dependency_tree_unit.test.ts index 1a92bb9894..8e9c733c00 100644 --- a/cli/test/dependency_tree_unit.test.ts +++ b/cli/test/dependency_tree_unit.test.ts @@ -2,7 +2,10 @@ import { expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import * as path from "node:path"; -import { DoubleLinkedDependencyTree } from "../src/utils/dependency_tree.ts"; +import { + DoubleLinkedDependencyTree, + resolvePlaceholdersFromLocal, +} from "../src/utils/dependency_tree.ts"; // addNode consults wmill-lock.yaml from cwd for workspace deps; run inside a // temp dir so the test never reads/writes the repo's own lock file. @@ -104,3 +107,62 @@ test("getAllTempScriptRefs is a superset of getTempScriptRefs for any node", asy }); }); }); + +// Two barrels deep so the fixpoint matters: resolving the first one is what +// puts the second in the tree, and only a further round reaches the leaf. +test("resolvePlaceholdersFromLocal walks the graph through unresolved barrels", async () => { + await withTempDir(async () => { + const tree = new DoubleLinkedDependencyTree(); + await tree.addNode( + "f/app/consumer", + `import { subtract } from "../barrel/index.ts"`, + "bun", + "", + ["f/barrel/index"], + "script", + "f/app/consumer", + "f/app/consumer.ts", + true, + ); + await tree.addNode( + "f/barrel/helper", + "export function subtract(a: number, b: number) { return a - b }", + "bun", + "", + [], + "script", + "f/barrel/helper", + "f/barrel/helper.ts", + true, + ); + // Neither barrel is in the change set, so both are bare import targets. + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({}); + + await resolvePlaceholdersFromLocal( + tree, + new Map([ + [ + "f/barrel/index", + { + localPath: "f/barrel/index.ts", + content: `export * from "./mid.ts"`, + }, + ], + [ + "f/barrel/mid", + { + localPath: "f/barrel/mid.ts", + content: `export * from "./helper.ts"`, + }, + ], + ]), + "bun", + ); + + // Only the leaf diverged from deployed, so only it was uploaded. + tree.setContentHash("f/barrel/helper", "hash_helper"); + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({ + "f/barrel/helper": "hash_helper", + }); + }); +}); diff --git a/cli/test/deploy_on_behalf_of_unit.test.ts b/cli/test/deploy_on_behalf_of_unit.test.ts index 2214c0dc37..7e21b24832 100644 --- a/cli/test/deploy_on_behalf_of_unit.test.ts +++ b/cli/test/deploy_on_behalf_of_unit.test.ts @@ -1,11 +1,12 @@ import { expect, test } from "bun:test"; import { deployItem } from "../windmill-utils-internal/src/deploy.ts"; -// `deployItem` spreads the source item into the request body, and a script's/flow's -// on_behalf_of names a username that only exists in the source -// workspace. Sending it to the target pairs one workspace's principal with the other's -// email, which the backend rejects. Deleting the spread is an easy regression, so pin -// that the key never reaches the wire. +// `deployItem` spreads the source item into the request body, and the principal it carries +// (`on_behalf_of`, at the top level for a script or flow and inside the policy for an app) +// names a username that only exists in the source workspace. Sending it to the target pairs +// one workspace's principal with the other's email, which the backend rejects. Deleting the +// spread is an easy regression, so pin that the principal never reaches the wire while the +// caller's chosen address does. function recordingProvider(captured: [string, any][], flowExists: boolean) { const source = { on_behalf_of_email: "alice@corp", @@ -32,6 +33,19 @@ function recordingProvider(captured: [string, any][], flowExists: boolean) { }), createScript: async (p: any) => void captured.push(["createScript", p.requestBody]), + existsApp: async () => false, + getAppByPath: async () => ({ + path: "f/x/a", + summary: "", + value: {}, + raw_app: false, + policy: { + execution_mode: "publisher", + on_behalf_of: "u/alice", + on_behalf_of_email: "alice@corp", + }, + }), + createApp: async (p: any) => void captured.push(["createApp", p.requestBody]), } as any; } @@ -65,19 +79,29 @@ test("deployItem: never sends the source workspace's on_behalf_of", async () => "dst", "alice@corp", ); + await deployItem( + recordingProvider(captured, false), + "app" as any, + "f/x/a", + "src", + "dst", + "alice@corp", + ); expect(captured.map(([fn]) => fn)).toEqual([ "createFlow", "updateFlow", "createScript", + "createApp", ]); - for (const [, body] of captured) { - // The email is still overridden with the caller's choice... - expect(body.on_behalf_of_email).toBe("alice@corp"); + for (const [name, body] of captured) { expect(body.preserve_on_behalf_of).toBe(true); + // Both surfaces spell it `on_behalf_of`; only its nesting differs — an app carries the + // identity inside its policy, the others at the top level. + const identity = name === "createApp" ? body.policy : body; + // The email is still overridden with the caller's choice... + expect(identity.on_behalf_of_email).toBe("alice@corp"); // ...while the principal is dropped, so the backend derives the target's own. - expect( - "on_behalf_of" in JSON.parse(JSON.stringify(body)), - ).toBe(false); + expect("on_behalf_of" in JSON.parse(JSON.stringify(identity))).toBe(false); } }); diff --git a/cli/test/gitsync_deploy_branch_config.test.ts b/cli/test/gitsync_deploy_branch_config.test.ts new file mode 100644 index 0000000000..39c435d872 --- /dev/null +++ b/cli/test/gitsync_deploy_branch_config.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createRemoteWorkspaceDeps, withTestBackend } from "./test_backend.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// The git-sync deploy callback starts in a clone of the tracked branch and +// switches to the fork's branch before pulling. What it writes there must follow +// that branch's wmill.yaml: a fork branch that turned on `dedupeLockfiles` keeps +// its shared lockfile instead of getting a `.script.lock` per script back. +test("git-sync fork deploy follows the fork branch's wmill.yaml", async () => { + await withTestBackend(async (backend) => { + const post = (path: string, body: unknown) => + backend.apiRequest!(`/api/w/${backend.workspace}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + await createRemoteWorkspaceDeps(backend, "python3", "wmill\n"); + await post("/folders/create", { name: "dedupe" }); + const script = await post("/scripts/create", { + path: "f/dedupe/a", + summary: "", + description: "", + content: "def main():\n pass\n", + language: "python3", + lock: "wmill==1.0.0\n", + }); + expect(script.ok).toBe(true); + + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_bare_")); + const seed = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_seed_")); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_work_")); + try { + execFileSync("git", ["init", "--bare", "--initial-branch=main", bare]); + git(seed, "init", "--initial-branch=main"); + git(seed, "config", "user.email", "seed@windmill.dev"); + git(seed, "config", "user.name", "seed"); + git(seed, "remote", "add", "origin", `file://${bare}`); + const wmillYaml = "defaultTs: bun\nincludes:\n - f/dedupe/**\nexcludes: []\n"; + await writeFile(join(seed, "wmill.yaml"), wmillYaml); + git(seed, "add", "-A"); + git(seed, "commit", "-m", "main"); + git(seed, "push", "origin", "main"); + + // Any parent id makes the callback deploy as a fork, to this branch. + const forkBranch = `wm-fork/main/${backend.workspace}`; + git(seed, "checkout", "-b", forkBranch); + await writeFile( + join(seed, "wmill.yaml"), + wmillYaml + "dedupeLockfiles: true\n", + ); + const pulled = await backend.runCLICommand(["sync", "pull", "--yes"], seed); + expect(pulled.code).toBe(0); + expect(existsSync(join(seed, "locks/requirements.in.lock"))).toBe(true); + git(seed, "add", "-A"); + git(seed, "commit", "-m", "dedupe"); + git(seed, "push", "origin", forkBranch); + + git(work, "clone", `file://${bare}`, "."); + const deployed = await backend.runCLICommand( + [ + "sync", + "git-deploy", + "--repository", + "u/test/unused", + "--git-deploy-items", + JSON.stringify([ + { path_type: "script", path: "f/dedupe/a", commit_msg: "deploy" }, + ]), + "--parent-workspace-id", + "parent", + ], + work, + ); + expect(deployed.code).toBe(0); + expect(git(work, "rev-parse", "--abbrev-ref", "HEAD")).toBe(forkBranch); + expect(git(work, "status", "--porcelain")).toBe(""); + } finally { + await rm(bare, { recursive: true, force: true }); + await rm(seed, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); + } + }); +}); diff --git a/cli/test/gitsync_deploy_push_unit.test.ts b/cli/test/gitsync_deploy_push_unit.test.ts new file mode 100644 index 0000000000..f87e86bcc3 --- /dev/null +++ b/cli/test/gitsync_deploy_push_unit.test.ts @@ -0,0 +1,98 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gitSyncDeployPush } from "../src/utils/git.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// A seeded clone of a bare remote, with `files` committed on main. +async function seededClone( + files: Record, +): Promise<{ bare: string; work: string }> { + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_push_bare_")); + execFileSync("git", ["init", "--quiet", "--bare", "--initial-branch=main", bare]); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_push_work_")); + git(work, "init", "--quiet", "--initial-branch=main"); + git(work, "config", "user.email", "seed@windmill.dev"); + git(work, "config", "user.name", "seed"); + for (const [path, content] of Object.entries(files)) { + await mkdir(join(work, path, ".."), { recursive: true }); + await writeFile(join(work, path), content); + } + git(work, "add", "-A"); + git(work, "commit", "--quiet", "-m", "seed"); + git(work, "remote", "add", "origin", `file://${bare}`); + git(work, "push", "--quiet", "-u", "origin", "main"); + return { bare, work }; +} + +function deployPushIn(work: string, path: string) { + const cwd = process.cwd(); + process.chdir(work); + try { + return gitSyncDeployPush({ + items: [{ path_type: "script", path, commit_msg: `deploy ${path}` }], + authorName: "windmill", + authorEmail: "windmill@windmill.dev", + }); + } finally { + process.chdir(cwd); + } +} + +test("a rewritten shared lockfile is committed with the deployed item", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline locks/requirements.in.lock'\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // What the deploy callback's pull leaves behind after `f/dd/a` was relocked: + // the item's own files are unchanged, only the shared file moved. + await writeFile(join(work, "locks/requirements.in.lock"), "requests==2.32.3\n"); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(work, "show", "--name-only", "--format=", "HEAD")).toBe( + "locks/requirements.in.lock", + ); + expect(git(bare, "cat-file", "-p", "main:locks/requirements.in.lock")).toBe( + "requests==2.32.3", + ); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a swept shared lockfile is committed as a deletion", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // The pull removed the last shared lockfile, and `locks/` with it. + await rm(join(work, "locks"), { recursive: true, force: true }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(bare, "ls-tree", "--name-only", "main", "locks/")).toBe(""); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a repository without shared lockfiles is left alone", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(false); + expect(git(bare, "rev-parse", "main")).toBe(git(work, "rev-parse", "HEAD")); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); diff --git a/cli/test/lint_orphan_metadata_unit.test.ts b/cli/test/lint_orphan_metadata_unit.test.ts new file mode 100644 index 0000000000..7066d10dca --- /dev/null +++ b/cli/test/lint_orphan_metadata_unit.test.ts @@ -0,0 +1,166 @@ +import { expect, test, describe } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import * as path from "node:path"; +import { runLint } from "../src/commands/lint/lint.ts"; + +const WMILL_YAML = "defaultTs: bun\nincludes:\n - f/**\nexcludes: []\n"; +const METADATA = "summary: test\nlock: ''\nschema:\n properties: {}\n"; + +async function write(dir: string, rel: string, content: string) { + const full = path.join(dir, rel); + await mkdir(path.dirname(full), { recursive: true }); + await writeFile(full, content, "utf-8"); +} + +/** + * Runs `fn` with a sync root at `/`, from which lint resolves + * every walked path. The name is a parameter because it is load-bearing: the + * folder suffixes lint classifies by (`.app`, `__mod`, …) are matched anywhere + * in a path, so a root carrying one must not change what lint reports. + */ +async function withSyncRoot( + rootName: string, + fn: (syncRoot: string) => Promise, + opts: { runFromParent?: boolean } = {}, +): Promise { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "wmill_lint_orphan_")); + const syncRoot = path.join(tempDir, rootName); + const originalCwd = process.cwd(); + try { + await write(syncRoot, "wmill.yaml", WMILL_YAML); + process.chdir(opts.runFromParent ? tempDir : syncRoot); + await fn(syncRoot); + } finally { + process.chdir(originalCwd); + await rm(tempDir, { recursive: true }); + } +} + +describe("orphan script metadata", () => { + test("reports metadata with no content file, with locks not required", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/paired.py", "def main():\n pass\n"); + await write(syncRoot, "f/paired.script.yaml", METADATA); + await write(syncRoot, "f/orphan.script.yaml", METADATA); + await write(syncRoot, "f/orphan_json.script.json", "{}\n"); + await write(syncRoot, "f/orphan_lock.script.lock", "some-dep==1.0.0\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.exitCode).toBe(1); + expect(report.issues.map((i) => i.path).sort()).toEqual([ + "f/orphan.script.yaml", + "f/orphan_json.script.json", + "f/orphan_lock.script.lock", + ]); + expect(report.issues[0].target).toBe("script"); + expect(report.issues[0].errors[0]).toContain("No script file found next to"); + }); + }); + + test("reports a module folder's own metadata with no content file", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/orphan__mod/script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual([ + "f/orphan__mod/script.yaml", + ]); + + // Linting the module folder itself: the walked paths no longer carry the + // `__mod/` boundary that says this is a module's metadata. + const inFolder = await runLint( + {} as any, + path.join(syncRoot, "f/orphan__mod"), + ); + + expect(inFolder.issues.map((i) => i.path)).toEqual(["script.yaml"]); + }); + }); + + test("reports orphans under a sync root named like a resource folder", async () => { + await withSyncRoot("acme.app", async (syncRoot) => { + await write(syncRoot, "f/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues.map((i) => i.path)).toEqual(["f/orphan.script.yaml"]); + }); + }); + + test("does not report a paired module under a sync root named like a module folder", async () => { + // Run from OUTSIDE the checkout, the one invocation whose paths carry the + // root's own name: nothing above the sync root may be classified. + await withSyncRoot( + "repo__mod", + async (syncRoot) => { + await write(syncRoot, "f/example__mod/script.yaml", METADATA); + await write( + syncRoot, + "f/example__mod/script.ts", + "export function main() {}\n", + ); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("does not report a non-dotted folder resource's child", async () => { + // The dotted/non-dotted setting is read from the invocation directory, so + // an explicit target configured the other way must still be recognized. + await withSyncRoot( + "repo", + async (syncRoot) => { + await write(syncRoot, "f/a__raw_app/raw_app.yaml", "value: {}\n"); + await write(syncRoot, "f/a__raw_app/backend/config.script.lock", "x\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }, + { runFromParent: true }, + ); + }); + + test("keeps the reported path whole when the lint target is a path segment", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/conf/orphan.script.yaml", METADATA); + + const report = await runLint({} as any, path.join(syncRoot, "f")); + + expect(report.issues.map((i) => i.path)).toEqual([ + "conf/orphan.script.yaml", + ]); + expect(report.issues[0].errors[0]).toContain("conf/orphan.script.yaml"); + }); + }); + + test("does not report a fileset child spelled like script metadata", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/data.resource.yaml", "value: {}\n"); + await write(syncRoot, "f/data.fileset/config.script.yaml", "a: 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); + + test("does not report a dbt project whose optional descriptor is absent", async () => { + await withSyncRoot("repo", async (syncRoot) => { + await write(syncRoot, "f/proj.script.yaml", METADATA); + await write(syncRoot, "f/proj__dbt/dbt_project.yml", "name: proj\n"); + await write(syncRoot, "f/proj__dbt/models/a.sql", "select 1\n"); + + const report = await runLint({} as any, syncRoot); + + expect(report.issues).toEqual([]); + }); + }); +}); diff --git a/cli/test/precheck_permissioned_as_apps_unit.test.ts b/cli/test/precheck_permissioned_as_apps_unit.test.ts new file mode 100644 index 0000000000..4a224a43c0 --- /dev/null +++ b/cli/test/precheck_permissioned_as_apps_unit.test.ts @@ -0,0 +1,147 @@ +/** + * The pre-check is what stops a push from silently reassigning an item's run-as + * user. Raw apps were missing from it, so the one kind whose file records no + * policy at all was also the one that changed owner without a word. + */ + +import { expect, test } from "bun:test"; +import { preCheckPermissionedAs } from "../src/core/permissioned_as.ts"; + +/** Non-interactive and without the override flag, the pre-check exits rather + * than reassigning silently — so a thrown exit is the signal it fired. */ +type Shape = "edited" | "added" | "deleted"; + +function change(path: string, name: Shape = "edited") { + return { name, path, before: "summary: x\n", content: "summary: x\n" }; +} + +async function precheck( + changes: ReturnType[], +): Promise { + const exit = process.exit; + let code: number | undefined; + (process as any).exit = (c?: number) => { + code = c; + throw new Error(`exit:${c}`); + }; + const logged: string[] = []; + const err = console.error; + console.error = (...a: unknown[]) => void logged.push(a.join(" ")); + try { + await preCheckPermissionedAs(changes, "pusher@corp", false, false, false); + } catch (e) { + if (!String(e).startsWith("Error: exit:")) throw e; + } finally { + (process as any).exit = exit; + console.error = err; + } + return code === undefined ? undefined : logged.join("\n"); +} + +test("a raw-app push warns the non-deployer it will take over the run-as user", async () => { + const message = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + + expect(message).toBeDefined(); + expect(message).toContain("f/test/myapp.raw_app"); + expect(message).toContain("pusher@corp"); +}); + +// Deleting one file re-pushes the whole app rather than deleting it, so the +// takeover happens there too. +test("deleting one of an app's files warns like editing one", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/gone.tsx", "deleted"), + ]); + + expect(message).toContain("f/test/myapp.raw_app"); +}); + +// The metadata file going with it means the app itself is created or removed — +// neither takes an owner over. +test("an app arriving or leaving whole is not a takeover", async () => { + const created = await precheck([ + change("f/test/new.raw_app/raw_app.yaml", "added"), + change("f/test/new.raw_app/index.tsx", "added"), + ]); + const removed = await precheck([ + change("f/test/old.raw_app/raw_app.yaml", "deleted"), + change("f/test/old.raw_app/index.tsx", "deleted"), + ]); + + expect(created).toBeUndefined(); + expect(removed).toBeUndefined(); +}); + +// An app carries no owner in its files, so nothing about it depends on their +// content — an empty one redeploys it exactly like any other. +test("an empty file still counts as a change to the app", async () => { + const added = await precheck([ + { name: "added", path: "f/test/myapp.raw_app/blank.ts", content: "" }, + ]); + const edited = await precheck([ + { name: "edited", path: "f/test/myapp.raw_app/blank.ts", before: "" }, + ]); + + expect(added).toContain("f/test/myapp.raw_app"); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `extractFolderPath` normalizes separators but the metadata predicates match a +// literal `/`, so a Windows path must not take a different branch. +test("a Windows path classifies the same as its posix twin", async () => { + const created = await precheck([ + change("f\\test\\new.raw_app\\raw_app.yaml", "added"), + change("f\\test\\new.raw_app\\index.tsx", "added"), + ]); + const edited = await precheck([ + change("f\\test\\myapp.raw_app\\index.tsx"), + ]); + + expect(created).toBeUndefined(); + expect(edited).toContain("f/test/myapp.raw_app"); +}); + +// `collectAppFiles` never sends these, and the sync diff never stops listing +// them (nothing uploads them, so they stay "added" forever) — so warning on one +// would gate every push of a scaffolded app on the override flag. +test("a file the push never sends is not a change to the app", async () => { + const artifacts = await precheck([ + change("f/test/myapp.raw_app/AGENTS.md", "added"), + change("f/test/myapp.raw_app/sql_to_apply/a.sql", "added"), + change("f/test/myapp.raw_app/node_modules/dep/index.js", "added"), + change("f/test/myapp.raw_app/recordings/r.json", "added"), + change("f/test/myapp.raw_app/package-lock.json"), + change("f/test/myapp.raw_app/wmill.d.ts"), + // Only the backend folder's *top level* is a runnable; nothing reads deeper, + // so the depth limit is what keeps a `backend/node_modules/` from becoming + // the perpetual diff this predicate exists to remove. + change("f/test/myapp.raw_app/backend/node_modules/dep/index.js", "added"), + ]); + // The three channels a push does send through: bundled file, metadata, runnable. + const sent = await precheck([change("f/test/myapp.raw_app/index.tsx")]); + const meta = await precheck([change("f/test/myapp.raw_app/raw_app.yaml")]); + const runnable = await precheck([change("f/test/myapp.raw_app/backend/a.ts")]); + // The runnable channel is not the bundle: the bundle's name exclusions don't + // reach into it, so a runnable file sharing one of those names still deploys. + const namesake = await precheck([ + change("f/test/myapp.raw_app/backend/wmill.d.ts"), + ]); + + expect(artifacts).toBeUndefined(); + expect(sent).toContain("f/test/myapp.raw_app"); + expect(meta).toContain("f/test/myapp.raw_app"); + expect(runnable).toContain("f/test/myapp.raw_app"); + expect(namesake).toContain("f/test/myapp.raw_app"); +}); + +test("an app is listed once however many of its files changed", async () => { + const message = await precheck([ + change("f/test/myapp.raw_app/index.tsx"), + change("f/test/myapp.raw_app/raw_app.yaml"), + change("f/test/myapp.raw_app/backend/a.ts"), + change("f/test/low.app/app.yaml"), + change("f/test/low.app/inline.ts"), + ]); + + expect(message).toContain("2 item(s)"); +}); diff --git a/cli/test/push_diff_convergence_unit.test.ts b/cli/test/push_diff_convergence_unit.test.ts new file mode 100644 index 0000000000..a6f2c132c0 --- /dev/null +++ b/cli/test/push_diff_convergence_unit.test.ts @@ -0,0 +1,348 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import JSZip from "jszip"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, sep } from "node:path"; +import { + checkoutInlineNames, + compareDynFSElement, + ZipFSElement, +} from "../src/commands/sync/sync.ts"; + +// The differ also reads the working tree (shared lockfiles, dependency files); +// an empty one keeps that out of the picture. +const originalCwd = process.cwd(); +beforeAll(() => { + process.chdir(mkdtempSync(join(tmpdir(), "wmill-push-diff-"))); +}); +afterAll(() => { + process.chdir(originalCwd); +}); + +// A push is only useful when a second run of it finds nothing left to do. +// These pin the two shapes that used to be listed on every run of a push into +// a fork while the push itself either applied nothing or aborted. + +type Mock = { + isDirectory: boolean; + path: string; + getContentText(): Promise; + getChildren(): AsyncIterable; +}; + +// Both sides of the differ use the OS separator; fixtures are written with +// "/" and rows are read back the same way. +const osPath = (p: string) => p.split("/").join(sep); +const slashPath = (p: string) => p.split(sep).join("/"); + +function local(files: Record): Mock { + return { + isDirectory: true, + path: "", + async getContentText() { + return ""; + }, + async *getChildren() { + for (const [path, content] of Object.entries(files)) { + yield { + isDirectory: false, + path: osPath(path), + async getContentText() { + return content; + }, + async *getChildren() {}, + }; + } + }, + }; +} + +const noIgnore = () => false; + +async function diff( + localEl: Mock, + remoteEl: Mock, + skips: Record, + parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, + isEls1Remote = false, +) { + const { changes } = await compareDynFSElement( + localEl as any, + remoteEl as any, + noIgnore, + false, + skips as any, + true, + [], + false, + undefined, + undefined, + isEls1Remote, + false, + parentOwnsScheduleEnabled, + ); + return changes.map((c) => `${c.name} ${slashPath(c.path)}`); +} + +const SCHEDULE = (enabled: string) => + `summary: nightly\nargs: {}\nenabled: ${enabled}\nis_flow: true\nschedule: 0 0 0 * * *\nscript_path: f/mail/flow\ntimezone: UTC\n`; + +test("push into a fork: a schedule the parent also has compares without `enabled`", async () => { + const remote = local({ "f/mail/nightly.schedule.yaml": SCHEDULE("true") }); + const skips = { includeSchedules: true }; + // The parent has `f/mail/nightly`; any other schedule is the fork's own. + const parentHas = (filePath: string) => + slashPath(filePath) === "f/mail/nightly.schedule.yaml"; + + // Not a fork: `enabled` is compared like any other field. + expect( + await diff( + local({ "f/mail/nightly.schedule.yaml": SCHEDULE("false") }), + remote, + skips, + ), + ).toEqual(["edited f/mail/nightly.schedule.yaml"]); + expect( + await diff( + local({ "f/mail/nightly.schedule.yaml": SCHEDULE("false") }), + remote, + skips, + parentHas, + ), + ).toEqual([]); + // The key being absent is the same case as it differing. + expect( + await diff( + local({ + "f/mail/nightly.schedule.yaml": SCHEDULE("false").replace( + "enabled: false\n", + "", + ), + }), + remote, + skips, + parentHas, + ), + ).toEqual([]); + // Only `enabled` is set aside. + expect( + await diff( + local({ + "f/mail/nightly.schedule.yaml": SCHEDULE("false").replace( + "0 0 0 * * *", + "0 0 1 * * *", + ), + }), + remote, + skips, + parentHas, + ), + ).toEqual(["edited f/mail/nightly.schedule.yaml"]); + // A schedule only the fork has keeps toggling from the file. + expect( + await diff( + local({ "f/mail/fork_only.schedule.yaml": SCHEDULE("true") }), + local({ "f/mail/fork_only.schedule.yaml": SCHEDULE("false") }), + skips, + parentHas, + ), + ).toEqual(["edited f/mail/fork_only.schedule.yaml"]); +}); + +const SUMMARY = "process one mail end-to-end (spam check, classify)"; + +function remoteFlow(content: string) { + const zip = new JSZip(); + zip.file( + "f/mail/flow_v2.flow.json", + JSON.stringify({ + summary: "Flow V2", + description: "", + value: { + modules: [ + { + id: "a", + summary: SUMMARY, + value: { + type: "rawscript", + content, + input_transforms: {}, + language: "python3", + }, + }, + ], + }, + schema: { type: "object", properties: {} }, + }), + // The backend's archive carries no directory entries. + { createFolders: false }, + ); + return zip; +} + +function localFlow(content: string) { + return local({ + "f/mail/flow_v2.flow/flow.yaml": `summary: Flow V2\ndescription: ''\nvalue:\n modules:\n - id: a\n summary: ${SUMMARY}\n value:\n type: rawscript\n content: '!inline process_mail.inline_script.py'\n input_transforms: {}\n language: python3\nschema:\n type: object\n properties: {}\n`, + "f/mail/flow_v2.flow/process_mail.inline_script.py": content, + }); +} + +// The checkout's `!inline` references, as `push` reads them from its flow.yaml. +const checkoutNames = async (flowDir: string) => + slashPath(flowDir) === "f/mail/flow_v2.flow" + ? { a: "process_mail.inline_script.py" } + : {}; + +test("push: an inline script the checkout names differently from the step summary is not a rename", async () => { + const skips = { includeSchedules: false }; + const render = (content: string, withCheckout: boolean) => + ZipFSElement( + remoteFlow(content), + true, + "bun", + {}, + {}, + false, + true, + withCheckout ? checkoutNames : undefined, + ) as any; + + // Same content, file named by hand: three rows before, none after. + expect( + await diff( + localFlow("def main():\n return 1\n"), + render("def main():\n return 1\n", false), + skips, + ), + ).toEqual([ + "deleted f/mail/flow_v2.flow/process_one_mail_end-to-end_(spam_check,_classify).inline_script.py", + "edited f/mail/flow_v2.flow/flow.yaml", + "added f/mail/flow_v2.flow/process_mail.inline_script.py", + ]); + expect( + await diff( + localFlow("def main():\n return 1\n"), + render("def main():\n return 1\n", true), + skips, + ), + ).toEqual([]); + + // A real edit is still one. + expect( + await diff( + localFlow("def main():\n return 2\n"), + render("def main():\n return 1\n", true), + skips, + ), + ).toEqual(["edited f/mail/flow_v2.flow/process_mail.inline_script.py"]); +}); + +test("push: a checkout name that collides with another step's summary-derived name keeps two files", async () => { + const zip = new JSZip(); + zip.file( + "f/mail/flow_v2.flow.json", + JSON.stringify({ + summary: "Flow V2", + description: "", + value: { + modules: [ + { + id: "a", + summary: SUMMARY, + value: { + type: "rawscript", + content: "a", + input_transforms: {}, + language: "python3", + }, + }, + { + id: "b", + summary: "process_mail", + value: { + type: "rawscript", + content: "b", + input_transforms: {}, + language: "python3", + }, + }, + ], + }, + schema: { type: "object", properties: {} }, + }), + { createFolders: false }, + ); + // Nothing local: every rendered file is a "deleted" row, one per path. + const rows = await diff( + local({}), + ZipFSElement(zip, true, "bun", {}, {}, false, true, checkoutNames) as any, + { includeSchedules: false }, + ); + expect(rows.filter((r) => r.endsWith(".py")).sort()).toEqual([ + "deleted f/mail/flow_v2.flow/process_mail.inline_script.py", + "deleted f/mail/flow_v2.flow/process_one_mail_end-to-end_(spam_check,_classify).inline_script.py", + ]); +}); + +test("push: checkout inline names stay inside the flow folder", async () => { + const flowYaml = join(process.cwd(), "flow.yaml"); + writeFileSync( + flowYaml, + `summary: x\nvalue:\n modules:\n - id: a\n value:\n type: rawscript\n content: '!inline a.inline_script.py'\n language: python3\n - id: b\n value:\n type: rawscript\n content: '!inline ../shared/b.py'\n language: python3\n - id: c\n value:\n type: rawscript\n content: '!inline /tmp/c.py'\n language: python3\n`, + ); + expect(await checkoutInlineNames(flowYaml)).toEqual({ + a: "a.inline_script.py", + }); + expect( + await checkoutInlineNames(join(process.cwd(), "missing.yaml")), + ).toEqual({}); +}); + +// A push never applies the workspace's display name and applies its color only +// when the local file carries one (see pushWorkspaceSettings), so a file that +// differs only in what would not be applied is not a push change; a pull still +// rewrites the file. +test("push: settings.yaml differing only by name or an unset color is not a change", async () => { + const remote = local({ + "settings.yaml": "name: prod\ncolor: '#ff0000'\nerror_handler: null\n", + }); + const unsetColor = local({ + "settings.yaml": "name: staging\nerror_handler: null\n", + }); + const skips = { includeSettings: true }; + expect(await diff(unsetColor, remote, skips)).toEqual([]); + expect(await diff(remote, unsetColor, skips, undefined, true)).toEqual([ + "edited settings.yaml", + ]); + + const otherColor = local({ + "settings.yaml": "name: staging\ncolor: '#00ff00'\nerror_handler: null\n", + }); + expect(await diff(otherColor, remote, skips)).toEqual(["edited settings.yaml"]); +}); + +// A push applies auto_invite.instance_groups only when the local file declares +// them (see pushWorkspaceSettings). +test("push: settings.yaml without instance_groups is not a change", async () => { + const remote = local({ + "settings.yaml": + "name: prod\nauto_invite:\n enabled: false\n instance_groups:\n - eng\n instance_groups_roles:\n eng: developer\n", + }); + const undeclared = local({ + "settings.yaml": "name: prod\nauto_invite:\n enabled: false\n", + }); + const skips = { includeSettings: true }; + expect(await diff(undeclared, remote, skips)).toEqual([]); + + const groupsOnlyRemote = local({ + "settings.yaml": "name: prod\nauto_invite:\n instance_groups:\n - eng\n", + }); + const noAutoInvite = local({ "settings.yaml": "name: prod\n" }); + expect(await diff(noAutoInvite, groupsOnlyRemote, skips)).toEqual([]); + + const otherGroups = local({ + "settings.yaml": + "name: prod\nauto_invite:\n enabled: false\n instance_groups: []\n", + }); + expect(await diff(otherGroups, remote, skips)).toEqual(["edited settings.yaml"]); +}); diff --git a/cli/test/push_workspace_settings_auto_invite_unit.test.ts b/cli/test/push_workspace_settings_auto_invite_unit.test.ts new file mode 100644 index 0000000000..b04d895056 --- /dev/null +++ b/cli/test/push_workspace_settings_auto_invite_unit.test.ts @@ -0,0 +1,82 @@ +/** + * Regression guard: `sync push` (pushWorkspaceSettings) applies the domain invite and + * the instance groups of `auto_invite` through their own endpoints, and never clears + * instance groups that settings.yaml does not declare. + */ + +import { expect, test, describe, beforeEach, mock } from "bun:test"; + +let editAutoInviteCalls: unknown[] = []; +let editInstanceGroupsCalls: unknown[] = []; +const remoteAutoInvite = { + enabled: true, + domain: "*", + operator: false, + mode: "invite", + instance_groups: ["eng"], + instance_groups_roles: { eng: "developer" }, +}; + +// Every wmill.* call reachable from pushWorkspaceSettings is stubbed: bun shares one +// mocked module across test files, and names missing from whichever mock loads first +// stay missing for the others. +mock.module("../gen/services.gen.ts", () => ({ + getSettings: async () => ({ auto_invite: remoteAutoInvite }), + getWorkspaceName: async () => "phoenix", + changeWorkspaceName: async () => {}, + changeWorkspaceColor: async () => {}, + editWebhook: async () => {}, + editAutoInvite: async (a: unknown) => { + editAutoInviteCalls.push(a); + }, + editInstanceGroups: async (a: unknown) => { + editInstanceGroupsCalls.push(a); + }, + editErrorHandler: async () => {}, + editSuccessHandler: async () => {}, + editCopilotConfig: async () => {}, + editLargeFileStorageConfig: async () => {}, + editWorkspaceGitSyncConfig: async () => {}, + editWorkspaceDefaultApp: async () => {}, + editDefaultScripts: async () => {}, + workspaceMuteCriticalAlertsUi: async () => {}, + updateOperatorSettings: async () => {}, + editDataTableConfig: async () => {}, + editSlackCommand: async () => {}, + setWorkspaceSlackOauthConfig: async () => {}, + deleteWorkspaceSlackOauthConfig: async () => {}, +})); + +const { pushWorkspaceSettings } = await import("../src/core/settings.ts"); + +describe("pushWorkspaceSettings auto_invite", () => { + beforeEach(() => { + editAutoInviteCalls = []; + editInstanceGroupsCalls = []; + }); + + test("an instance-group-only change updates the groups and leaves the domain invite", async () => { + await pushWorkspaceSettings("phoenix", "settings", undefined, { + name: "phoenix", + auto_invite: { ...remoteAutoInvite, instance_groups_roles: { eng: "admin" } }, + }); + expect(editAutoInviteCalls.length).toBe(0); + expect(editInstanceGroupsCalls).toEqual([ + { + workspace: "phoenix", + requestBody: { groups: ["eng"], roles: { eng: "admin" } }, + }, + ]); + }); + + test("a settings.yaml without instance_groups does not clear them", async () => { + const { instance_groups: _g, instance_groups_roles: _r, ...domainInvite } = + remoteAutoInvite; + await pushWorkspaceSettings("phoenix", "settings", undefined, { + name: "phoenix", + auto_invite: { ...domainInvite, operator: true }, + }); + expect(editAutoInviteCalls.length).toBe(1); + expect(editInstanceGroupsCalls.length).toBe(0); + }); +}); diff --git a/cli/test/push_workspace_settings_name_unit.test.ts b/cli/test/push_workspace_settings_identity_unit.test.ts similarity index 53% rename from cli/test/push_workspace_settings_name_unit.test.ts rename to cli/test/push_workspace_settings_identity_unit.test.ts index ef76ab7aec..5dcf9815ef 100644 --- a/cli/test/push_workspace_settings_name_unit.test.ts +++ b/cli/test/push_workspace_settings_identity_unit.test.ts @@ -1,27 +1,37 @@ /** - * Regression guard: a pull (pushWorkspaceSettings) must not apply the workspace - * display name from settings.yaml. Rationale lives at the apply site in settings.ts. + * Regression guard: `sync push` (pushWorkspaceSettings) must never apply the + * workspace display name from settings.yaml, and must apply the color only when + * the file carries one. Rationale lives at the apply sites in settings.ts. */ import { expect, test, describe, beforeEach, mock } from "bun:test"; let changeWorkspaceNameCalls: unknown[] = []; +let changeWorkspaceColorCalls: unknown[] = []; let editWebhookCalls: unknown[] = []; let remoteName = ""; +let remoteColor: string | undefined = undefined; let remoteWebhook: string | undefined = undefined; // Every wmill.* call reachable from pushWorkspaceSettings is stubbed so the -// function runs without a backend; only the two we assert on record calls. +// function runs without a backend; only the three we assert on record calls. mock.module("../gen/services.gen.ts", () => ({ - getSettings: async (_a: { workspace: string }) => ({ webhook: remoteWebhook }), + getSettings: async (_a: { workspace: string }) => ({ + webhook: remoteWebhook, + color: remoteColor, + }), getWorkspaceName: async (_a: { workspace: string }) => remoteName, changeWorkspaceName: async (a: unknown) => { changeWorkspaceNameCalls.push(a); }, + changeWorkspaceColor: async (a: unknown) => { + changeWorkspaceColorCalls.push(a); + }, editWebhook: async (a: unknown) => { editWebhookCalls.push(a); }, editAutoInvite: async () => {}, + editInstanceGroups: async () => {}, editErrorHandler: async () => {}, editSuccessHandler: async () => {}, editCopilotConfig: async () => {}, @@ -30,7 +40,6 @@ mock.module("../gen/services.gen.ts", () => ({ editWorkspaceDefaultApp: async () => {}, editDefaultScripts: async () => {}, workspaceMuteCriticalAlertsUi: async () => {}, - changeWorkspaceColor: async () => {}, updateOperatorSettings: async () => {}, editDataTableConfig: async () => {}, editSlackCommand: async () => {}, @@ -40,13 +49,15 @@ mock.module("../gen/services.gen.ts", () => ({ const { pushWorkspaceSettings } = await import("../src/core/settings.ts"); -describe("pushWorkspaceSettings workspace name", () => { +describe("pushWorkspaceSettings workspace identity", () => { const ws = "phoenix"; beforeEach(() => { changeWorkspaceNameCalls = []; + changeWorkspaceColorCalls = []; editWebhookCalls = []; remoteName = "phoenix"; + remoteColor = undefined; remoteWebhook = undefined; }); @@ -69,4 +80,37 @@ describe("pushWorkspaceSettings workspace name", () => { expect(editWebhookCalls.length).toBe(0); expect(changeWorkspaceNameCalls.length).toBe(0); }); + + test("a settings.yaml without a color key does not clear the workspace color", async () => { + remoteColor = "#ff0000"; + remoteWebhook = "https://old"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + webhook: "https://new", + }); + expect(editWebhookCalls.length).toBe(1); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); + + test("a color in settings.yaml is applied when it differs from the workspace", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#00ff00", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls).toEqual([ + { workspace: ws, requestBody: { color: "#00ff00" } }, + ]); + }); + + test("a color matching the workspace is a complete no-op", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#ff0000", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); }); diff --git a/cli/test/raw_app_push_policy_unit.test.ts b/cli/test/raw_app_push_policy_unit.test.ts new file mode 100644 index 0000000000..fbbc7b55c0 --- /dev/null +++ b/cli/test/raw_app_push_policy_unit.test.ts @@ -0,0 +1,142 @@ +/** + * `raw_app.yaml` records none of the policy but the access-mode markers, so a + * push that regenerated the whole policy reset the deploy drawer's settings — + * run-as identity, sandbox isolation — to the pushing user's. Pin that the + * deployed policy is carried over, that a first push still starts from what the + * file states, and that the markers still close a deployed open app back down. + */ + +import { afterAll, beforeEach, expect, mock, test } from "bun:test"; +import { mkdtemp, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let calls: any[] = []; +let deployedPolicy: any; +/** No app deployed at the path: `getAppByPath` 404s and the push creates one. */ +let deployed = true; + +// Stub only what no other in-process suite imports, and treat a stub as +// permanent for the run (see "Module mocks" in cli/TESTING.md). These three API +// functions qualify — nothing else in `test/` imports them. `bundle.ts` did not: +// stubbing it left `raw_app_svelte_plugin_unit.test.ts` asserting against an +// empty bundle, which an `afterAll` hand-back did not prevent. So the real +// bundler runs instead, on the app each push writes below. +const realServices = await import("../gen/services.gen.ts"); + +mock.module("../gen/services.gen.ts", () => ({ + ...realServices, + getAppByPath: async () => { + if (!deployed) throw new Error("not found"); + return { + path: "f/test/raw", + summary: "raw", + value: { files: {}, runnables: {} }, + policy: deployedPolicy, + }; + }, + updateAppRaw: async (a: unknown) => { + calls.push(a); + }, + createAppRaw: async (a: unknown) => { + calls.push(a); + }, +})); + +// Belt and braces: nothing else in-process calls these, and a hand-back is not +// what makes that safe. +afterAll(() => { + mock.module("../gen/services.gen.ts", () => realServices); +}); + +const { pushRawApp } = await import("../src/commands/app/raw_apps.ts"); + +const ADMIN = { + userCache: new Map(), + userIsAdminOrDeployer: true, + userEmail: "deployer@windmill.dev", +}; + +async function push(yamlTail: string, admin = true): Promise { + calls = []; + const dir = await mkdtemp(join(tmpdir(), "windmill_raw_push_")); + await writeFile( + join(dir, "raw_app.yaml"), + `summary: raw\nrunnables: {}\n${yamlTail}`, + "utf-8", + ); + // Any file the remote doesn't have, so the push isn't short-circuited as + // up to date. It is also the bundler's entry point, so it has to compile. + await writeFile(join(dir, "index.tsx"), "export default 1\n", "utf-8"); + await writeFile( + join(dir, "package.json"), + JSON.stringify({ name: "app", private: true }), + "utf-8", + ); + // `ensureNodeModules` only checks the directory is there; borrowing the CLI's + // own skips an npm install per push. + await symlink(join(process.cwd(), "node_modules"), join(dir, "node_modules")); + await pushRawApp("w", "f/test/raw", dir, undefined, "bun", admin ? ADMIN : undefined); + expect(calls).toHaveLength(1); + return calls[0].formData.app; +} + +beforeEach(() => { + deployed = true; + deployedPolicy = { + on_behalf_of: "u/svc", + on_behalf_of_email: "svc@corp", + sandbox: true, + frontend_sdk_scopes: ["jobs:run"], + execution_mode: "anonymous", + // Legacy v1 grants: the backend folds them into v2 at run time, so keeping + // them would keep granting runnables a push has removed. + triggerables: { "script/f/test/gone": {} }, + triggerables_v2: { "a:script/f/test/gone": {} }, + }; +}); + +test("a raw-app push keeps the deployed run-as and sandbox settings", async () => { + const body = await push("public: true\n"); + + expect(body.policy.on_behalf_of).toBe("u/svc"); + expect(body.policy.on_behalf_of_email).toBe("svc@corp"); + expect(body.preserve_on_behalf_of).toBe(true); + expect(body.policy.sandbox).toBe(true); + expect(body.policy.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(body.policy.execution_mode).toBe("anonymous"); + expect(body.policy.triggerables).toBeUndefined(); + expect(body.policy.triggerables_v2).toEqual({}); +}); + +test("a raw-app push without the marker closes an anonymous app back down", async () => { + const body = await push(""); + + expect(body.policy.execution_mode).toBe("publisher"); +}); + +test("a push that may not claim the deployed identity doesn't send it", async () => { + const body = await push("", false); + + expect(body.preserve_on_behalf_of).toBeUndefined(); + // Not just the flag: the identity itself stays off the wire, so no server can + // deploy this push under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + // Everything the pusher is entitled to carry over still comes along. + expect(body.policy.sandbox).toBe(true); +}); + +test("a first raw-app push deploys the policy its file states", async () => { + deployed = false; + const body = await push( + "policy:\n sandbox: true\n on_behalf_of: u/impostor\n on_behalf_of_email: impostor@corp\n", + ); + + expect(body.policy.sandbox).toBe(true); + // A repo doesn't get to pick who an app runs as: the identity never reaches + // the wire, so no server can be talked into deploying under it. + expect(body.policy.on_behalf_of).toBeUndefined(); + expect(body.policy.on_behalf_of_email).toBeUndefined(); + expect(body.preserve_on_behalf_of).toBeUndefined(); +}); diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..17a1731494 --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,143 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, afterAll, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +// A module mock is process-global and outlives the file that installs it, and +// `mock.restore()` does not undo one: every mocked module has to be handed back +// its real exports here, or whichever file `bun test` happens to run next gets +// this file's stubs. +const realModules: [string, Record][] = []; +async function mockModule( + specifier: string, + factory: (real: Record) => Record +): Promise { + const real = { ...((await import(specifier)) as Record) }; + realModules.push([specifier, real]); + mock.module(specifier, () => factory(real)); +} + +afterAll(() => { + for (const [specifier, real] of realModules) { + mock.module(specifier, () => real); + } +}); + +await mockModule("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +await mockModule("../src/core/context.ts", (real) => ({ + ...real, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +await mockModule("../src/core/auth.ts", (real) => ({ + ...real, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/secret_bearing_deletions_unit.test.ts b/cli/test/secret_bearing_deletions_unit.test.ts new file mode 100644 index 0000000000..1fe2603527 --- /dev/null +++ b/cli/test/secret_bearing_deletions_unit.test.ts @@ -0,0 +1,66 @@ +/** + * The trashbin notice a push prints is only as good as its classification, which has + * to agree with the apply loop on two things: which deleted files are a variable or a + * resource — a path the loop skips must not be counted, or the notice announces a + * deletion that never happened — and that the unit is the server-side object, so a + * file resource's two files are the one deletion they cause. + */ + +import { describe, expect, test } from "bun:test"; +import { + secretBearingObjectKind, + describeSecretBearingChanges, +} from "../src/commands/sync/sync.ts"; + +describe("secretBearingObjectKind", () => { + test("matches variable and resource metadata in both serializations", () => { + expect(secretBearingObjectKind("f/test/protocol.variable.yaml")).toBe( + "variable", + ); + expect(secretBearingObjectKind("f/test/erp_access.resource.json")).toBe( + "resource", + ); + // A file resource's content file deletes the resource outright, so it counts. + expect(secretBearingObjectKind("f/test/conf.resource.file.ini")).toBe( + "resource", + ); + }); + + test("ignores files that only look like one", () => { + for (const p of [ + "f/test/my_type.resource-type.json", + // A fileset child can be any file; deleting one re-pushes the parent resource + // rather than deleting anything. + "f/test/data.fileset/edge/inner.resource.yaml", + "f/test/bar.script.yaml", + "f/test/foo.flow/flow.yaml", + // The apply loop skips a `.lock` deletion outright, so counting one would + // announce a deletion that never happens. Reachable for a resource type whose + // format_extension is literally `lock`. + "f/test/conf.resource.file.lock", + ]) { + expect(secretBearingObjectKind(p)).toBeUndefined(); + } + }); +}); + +describe("describeSecretBearingChanges", () => { + test("counts each kind separately and pluralizes", () => { + expect( + describeSecretBearingChanges([ + { path: "f/a.variable.yaml" }, + { path: "f/b.variable.yaml" }, + { path: "f/c.resource.yaml" }, + ]), + ).toBe("2 variables and 1 resource"); + }); + + test("counts a file resource's two files as the one resource they delete", () => { + expect( + describeSecretBearingChanges([ + { path: "f/c.resource.yaml" }, + { path: "f/c.resource.file.ini" }, + ]), + ).toBe("1 resource"); + }); +}); diff --git a/cli/test/shared_ui_diff_unit.test.ts b/cli/test/shared_ui_diff_unit.test.ts index f2fd69210e..052fa58a8e 100644 --- a/cli/test/shared_ui_diff_unit.test.ts +++ b/cli/test/shared_ui_diff_unit.test.ts @@ -11,12 +11,25 @@ import * as os from "node:os"; import * as path from "node:path"; let remoteFiles: Record = {}; +let remoteUnreadable = false; +let pushedFiles: Record | undefined; mock.module("../gen/services.gen.ts", () => ({ - getSharedUi: async (_args: { workspace: string }) => ({ files: remoteFiles }), + getSharedUi: async (_args: { workspace: string }) => { + if (remoteUnreadable) throw new Error("shared UI store unreadable"); + return { files: remoteFiles }; + }, + updateSharedUi: async (args: { + workspace: string; + requestBody: { files: Record }; + }) => { + pushedFiles = args.requestBody.files; + }, })); -const { diffSharedUi } = await import("../src/commands/shared_ui.ts"); +const { diffSharedUi, pushSharedUi } = await import( + "../src/commands/shared_ui.ts" +); describe("diffSharedUi", () => { const ws = "test-workspace"; @@ -25,6 +38,8 @@ describe("diffSharedUi", () => { beforeEach(() => { remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; prevCwd = process.cwd(); tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-")); process.chdir(tmpDir); @@ -89,4 +104,46 @@ describe("diffSharedUi", () => { const changes = await diffSharedUi(ws); expect(changes).toEqual([]); }); + + test("emits nothing under keepDeleted when the remote store is unreadable", async () => { + // pushSharedUi skips the push rather than clearing a store it can't read, + // so the preview must show that same nothing. + remoteUnreadable = true; + writeUi("theme.json", "{}"); + expect(await diffSharedUi(ws, true)).toEqual([]); + }); +}); + +describe("pushSharedUi with keepDeleted", () => { + const ws = "test-workspace"; + let tmpDir: string; + let prevCwd: string; + + beforeEach(() => { + remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; + prevCwd = process.cwd(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-push-")); + process.chdir(tmpDir); + }); + + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + test("carries remote-only files, including ui/__proto__, into the pushed map", async () => { + // JSON.parse, not a literal: `{__proto__: …}` sets the prototype instead of + // creating the own property the API response really has. + remoteFiles = JSON.parse('{"__proto__":"keep me","extra.json":"1"}'); + fs.mkdirSync(path.join(tmpDir, "ui"), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, "ui", "theme.json"), "{}", "utf-8"); + + expect(await pushSharedUi(ws, true)).toBe(true); + // The store is written whole, so anything missing here is deleted. + expect(pushedFiles!["extra.json"]).toEqual("1"); + expect(Object.getOwnPropertyDescriptor(pushedFiles!, "__proto__")?.value) + .toEqual("keep me"); + }); }); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 343ed49ef7..24c4a282a3 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2774,3 +2774,102 @@ kind: script }); }); }); + +describe("keep deleted", () => { + test("Integration: --keep-deleted keeps items absent from the other side", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/keep_deleted_${uniqueId}`; + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: 'export async function main() { return "keep me"; }', + language: "bun", + summary: "Kept by --keep-deleted", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + await writeWmillYaml(tempDir); + expect( + (await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code + ).toEqual(0); + + const contentFile = `${scriptPath}.ts`; + const metadataFile = `${scriptPath}.script.yaml`; + expect(await listFilesRecursive(tempDir)).toContain(contentFile); + + // Push direction: the remote script survives losing its local files. + await rm(join(tempDir, contentFile)); + await rm(join(tempDir, metadataFile)); + expect( + ( + await backend.runCLICommand( + ["sync", "push", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + // A push deletion archives the script rather than removing the row, so + // `archived` — not the status code — is what says it survived. + const remote = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}` + ); + expect(remote.status).toEqual(200); + expect((await remote.json()).archived).not.toEqual(true); + + // Pull direction: a file with no remote counterpart survives the pull. + const localOnly = `f/test/local_only_${uniqueId}.ts`; + await writeFile( + join(tempDir, localOnly), + 'export async function main() { return "local only"; }', + "utf-8" + ); + expect( + ( + await backend.runCLICommand( + ["sync", "pull", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + const afterPull = await listFilesRecursive(tempDir); + expect(afterPull).toContain(localOnly); + // Adds still apply: the script deleted above is written back. + expect(afterPull).toContain(contentFile); + + // An empty changeset falls past the dry-run return, on to the shared-UI + // step — which writes to disk, so a dry run must skip it. + // Including the metadata and lock the pull's auto-fill generated for it. + for (const ext of [".ts", ".script.yaml", ".script.lock"]) { + await rm(join(tempDir, `f/test/local_only_${uniqueId}${ext}`), { + force: true, + }); + } + await mkdir(join(tempDir, "ui"), { recursive: true }); + await writeFile(join(tempDir, "ui", "custom.css"), "body{}", "utf-8"); + const dryRun = await backend.runCLICommand( + ["sync", "pull", "--dry-run"], + tempDir + ); + expect(dryRun.code).toEqual(0); + // Guards against a vacuous pass: a non-empty changeset would return at + // the dry-run check above and never reach the shared-UI step. + expect(dryRun.stdout + dryRun.stderr).toContain("0 changes to apply"); + expect(await listFilesRecursive(tempDir)).toContain("ui/custom.css"); + }); + }); +}); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index e9ea167ffc..38da094a37 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -168,7 +168,7 @@ test( // Customer scenario: a barrel file (f/lib/errors/index.ts) re-exports from // siblings (./types.ts, ./WorkflowError.ts, ...). An importer in a different // folder imports from the barrel. On a fresh DB, the dep job for the importer -// fetches index.ts via raw_unpinned + temp_script_hash, but bun's resolver +// fetches index.ts via raw + temp_script_hash, but bun's resolver // then has to resolve the barrel's *sibling* imports — and those need to be // in TEMP_SCRIPT_REFS too. test( @@ -268,3 +268,78 @@ test( }); }, ); + +// The importer and the leaf change; the barrel between them does not, so it is +// absent from the push's change set. See `resolvePlaceholdersFromLocal`. +test( + "sync push --auto-metadata succeeds when a changed leaf sits behind an unchanged barrel", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + + await createLocalScript( + tempDir, + "f/barrel", + "helper", + "bun", + `export function add(a: number, b: number) { return a + b; }\n`, + ); + await createLocalScript( + tempDir, + "f/barrel", + "index", + "bun", + `export * from "./helper.ts";\n`, + ); + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { add } from "../barrel/index.ts"; +export async function main() { return add(1, 2); } +`, + ); + + const deploy = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (deploy.code !== 0) { + console.log("STDOUT:", deploy.stdout); + console.log("STDERR:", deploy.stderr); + } + expect(deploy.code).toBe(0); + + // Add an export to the leaf and use it from the importer. The barrel + // re-exports it already, so it stays byte-identical and out of the push. + await writeFile( + `${tempDir}/f/barrel/helper.ts`, + `export function add(a: number, b: number) { return a + b; } +export function subtract(a: number, b: number) { return a - b; } +`, + ); + await writeFile( + `${tempDir}/f/app/consumer.ts`, + `import { subtract } from "../barrel/index.ts"; +export async function main() { return subtract(3, 1); } +`, + ); + + const result = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("No matching export"); + expect(combined).not.toContain("Failed to generate lockfile"); + }); + }, +); diff --git a/cli/test/trash_commands.test.ts b/cli/test/trash_commands.test.ts new file mode 100644 index 0000000000..7b417dbc5e --- /dev/null +++ b/cli/test/trash_commands.test.ts @@ -0,0 +1,66 @@ +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { setupWorkspaceProfile, ensureFolder } from "./new_commands_helpers.ts"; + +describe("trash command", () => { + test("lists, shows and restores a deleted variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + await ensureFolder(backend, "test"); + const ws = backend.workspace; + const path = `f/test/trash_${Date.now()}`; + const api = (route: string, init: RequestInit = {}) => + backend.apiRequest!(`/api/w/${ws}/${route}`, { + headers: { "Content-Type": "application/json" }, + ...init, + }); + + let resp = await api("variables/create", { + method: "POST", + body: JSON.stringify({ path, value: "kept", is_secret: false, description: "" }), + }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + resp = await api(`variables/delete/${path}`, { method: "DELETE" }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + const list = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(list.code).toBe(0); + const item = JSON.parse(list.stdout).find((i: any) => i.item_path === path); + expect(item).toBeDefined(); + expect(item.item_kind).toBe("variable"); + + const get = await backend.runCLICommand( + ["trash", "get", "--json", String(item.id)], + tempDir + ); + expect(get.code).toBe(0); + expect(JSON.parse(get.stdout).item_data.row.value).toBe("kept"); + + // A bogus second id: the first restore must still go through, and the + // failure must show in the exit code. + const restore = await backend.runCLICommand( + ["trash", "restore", String(item.id), "999999999"], + tempDir + ); + expect(restore.code).toBe(1); + expect(restore.stdout).toContain(`variable '${path}' restored`); + expect(restore.stderr).toContain("999999999"); + + resp = await api(`variables/get/${path}`); + expect(resp.status).toBe(200); + expect((await resp.json()).value).toBe("kept"); + + const after = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(after.code).toBe(0); + expect(JSON.parse(after.stdout).some((i: any) => i.item_path === path)).toBe(false); + }); + }); +}); diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index 82b7bd7f36..31a605cbf9 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -423,6 +423,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index fb16a0b068..054ccbba40 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -506,10 +506,23 @@ export async function deployItem( }, }); } else if (kind === "app" || kind === "raw_app") { - const app = await provider.getAppByPath({ + const rawApp = await provider.getAppByPath({ workspace: workspaceFrom, path, }); + // See the flow branch: a source-workspace principal is never valid here, and the + // policy carries the app's in `on_behalf_of`. Clearing it lets the backend derive + // the target's own from the address. A group travels as its synthetic + // `group-*@windmill.dev` address, which an admin-created account holding it would + // win: known and accepted, see `users::permissioned_as_from_email` in the backend. + const app = { + ...rawApp, + policy: { + ...rawApp.policy, + on_behalf_of: undefined, + on_behalf_of_email: onBehalfOf, + }, + }; if (alreadyExists) { if (app.raw_app) { const secret = await provider.getPublicSecretOfLatestVersionOfApp({ diff --git a/docker/DockerfileExtra b/docker/DockerfileExtra index 9eb2a987f2..8aa24206a4 100644 --- a/docker/DockerfileExtra +++ b/docker/DockerfileExtra @@ -110,11 +110,12 @@ WORKDIR /multiplayer # Copy multiplayer server files COPY multiplayer/package.json . +COPY multiplayer/package-lock.json . COPY multiplayer/server.mjs . COPY multiplayer/gateway.mjs . # Install dependencies -RUN npm install +RUN npm ci --omit=dev # ============================================================================ # Entrypoint Setup diff --git a/docker/RHEL8/Dockerfile b/docker/RHEL8/Dockerfile index c17c7685c2..ea53feabf1 100644 --- a/docker/RHEL8/Dockerfile +++ b/docker/RHEL8/Dockerfile @@ -33,6 +33,8 @@ COPY /backend/oauth_connect.json /backend/oauth_connect.json COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh COPY /system_prompts/auto-generated /system_prompts/auto-generated +# The flow chat imports the chat SDK's source (svelte.config.js alias `windmill-chat`). +COPY /chat-sdk/src /chat-sdk/src RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index 3f24e936ab..ab94df5fc4 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -33,6 +33,8 @@ COPY /backend/oauth_connect.json /backend/oauth_connect.json COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh COPY /system_prompts/auto-generated /system_prompts/auto-generated +# The flow chat imports the chat SDK's source (svelte.config.js alias `windmill-chat`). +COPY /chat-sdk/src /chat-sdk/src RUN cd /backend/windmill-api && . ./build_openapi.sh COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md new file mode 100644 index 0000000000..ffec41e182 --- /dev/null +++ b/docs/ai-session-backups.md @@ -0,0 +1,400 @@ +# AI session backups + +AI sessions live in the browser: the session list (`windmill-sessions`), chat transcripts and +image blobs (`copilot-chat-history`) and artifacts (`copilot-artifacts`), all per-user IndexedDB +stores. This is the design of their backup in the workspace's object storage, and the +constraints future work on either side must keep. + +Backend: `backend/windmill-api/src/ai_sessions.rs` (`/w/{w}/ai/sessions/{list,pull,push}`). +Frontend: `frontend/src/lib/components/sessions/sessionMirror*.ts`. + +## Why it is lazy + +A session changes at the local write rate: a transcript write every 2 s while streaming, a +session-record write per new message on screen. An object in S3 is replaced whole and every PUT +is billed, so the backup deliberately does not follow that rate. Local writes only mark a session +dirty (`sessionMirrorSignal.ts`, import-free so the stores never depend on the backup). A flush +runs 15 s after the marks go quiet, at most 2 min after the first unflushed mark, when the tab is +hidden, and 10 s after load for marks a crash left behind. Marks are persisted in localStorage +(shared by the user's tabs) for that reason, one key per mark: a shared blob would let two tabs +marking different sessions at once rewrite each other's mark away. A dirty mark is a counter +bumped on every write; a push retires it by recording the counter it covered on the session's +sync row rather than deleting the mark, since two localStorage calls cannot compare-and-delete +and a bump landing between them would be lost; retired marks are not reclaimed (one small key per +session ever backed up), and the marks of unsent drafts and of workspaces that are off stay too, +each costing one lookup per flush. Only a session gone from the store has its mark deleted. Losing the last +seconds of a device that never comes back is accepted; a tab that closes normally keeps its marks. +A signal names the user whose store the write landed in (read off the store's scoped name), so +a write that completes after the logged-in user changed marks that user's session, for their +next load, rather than the current user's. + +A flush plans and sends one session at a time, filling requests of about 8 MB as it goes, so a +first backfill of a large history never holds more than one request's worth of records and +images in memory. + +## What a push carries + +The pure planner (`sessionMirrorPlan.ts`) compares each piece against the marker of what was +last pushed, kept per session in the `windmill-sessions-mirror` store: + +| Piece | Object | Sent when | +|---|---|---| +| session record | `sessions/{sid}/head.json` | its signature changed | +| chat | `sessions/{sid}/chats/{cid}.json` | its `lastModified` moved | +| artifacts | `sessions/{sid}/artifacts.json` | their fingerprint changed | +| image | `images/{sid}/{cid}/{iid}` | never pushed before (write-once) | +| index marker | `index/{sid}/{epoch}` | last, by the part that completes a push of the session (empty; named by the record's move count) | + +All under `windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/` in the workspace's +primary storage (the generation is what a key rotation moves, see below). +The listing reads only `index/`: one object per session whatever the session holds, so a +session with many chats cannot crowd newer ones out of a bounded scan, and its +`last_modified` is the session's `updated_at`. Written last, and only by an entry no unsent +part follows (a session split over several entries says `partial` on all but the last, and +names the push on each, or the part is refused), it +lists a session only once a whole push landed; the parts of a session after a failed one are +not written either, on the server within one push and on the client across pushes, so the +marker on the last part never lists a session missing a chat, and a new session whose last part +never lands is not listed at all. A push of the session whole (no sync row, or a stale one, +`whole` on every part) opens with the head on the first: that part replaces the backup (the +marker goes first, then everything under the session), so what an old storage still held of +the session and the push does not carry is gone. An incremental part rides on a listed +session, and the server refuses it with `needs_whole`, writing nothing, when none is listed +(a removal deletes the marker first), rather than write a marker over a session missing what +earlier parts or earlier pushes carried; one that changes more than one object unlists the +session before its writes and lists it again after them, so a pull between two of the writes, +or after one failed, finds it absent rather than a mix of old and new pieces (one object +changing is one write, and stays listed). A push split over several parts, whole or +incremental, names itself on each with a token the browser draws (`push`, `opens` on the +first): the opening part unlists the session, so a pull between two parts finds it absent +rather than a mix of old and new pieces, the last part lists it again, and a later part is +written only while that token is the one there, so two devices pushing the session at once +cannot list a mix of their pieces (the push that opened later wins; the other is refused with +`needs_whole` and goes again), and one abandoned leaves the session unlisted, so the next +push of it goes whole. A push and a removal of one session +are serialized on the server by a Postgres advisory lock keyed on the session's prefix, so +the two never interleave object by object. + +The head signature leaves out `name` (a per-browser counter the sessions page routes by), +the unsent-draft fields, `workspace_root_id` (recomputed on import), and the two fields reading +a session bumps (`lastSeenCount`, `lastActivityAt`). Reading a session must never cost a push; +keep that property when adding fields to `Session`. + +Unsent drafts (no `workspace_id`) and attached files (Blobs, directory handles) are not backed up. + +## Encryption and access + +Every object is encrypted with a key derived from the workspace key and the user +(`build_crypt_with_key_suffix` with the email hash), because workspace storage credentials are +shared far more widely than a user's transcripts: `public_resource` storages and legacy-mode +READ/WRITE hand any member the bucket. The key is per user rather than per workspace so that a +member who copies another user's ciphertext under their own prefix gets nothing from `pull`; an +object that does not decrypt for its reader is treated as absent. Rotating the workspace key +(`set_encryption_key`) does not re-key the backups the way it re-encrypts the workspace's +secrets. The objects live under a prefix named by a generation +(`workspace_settings.ai_sessions_backup_generation`) that the rotation bumps in the +transaction committing the new key; once committed, the routes read and write under the new +generation's prefix, the answers name it (`backup_generation`, below; `storage_id` names the +storage and does not change), so every browser marks its sync rows stale and pushes its +sessions whole again there, and every older generation, which nothing writes to any more, is deleted off the +request at leisure (`windmill-api-workspaces/src/ai_session_backups.rs`). Sessions no browser +holds any more are lost. A generation is never reused, so no deletion, however late, can +touch live objects; a rotation that fails before its commit bumps nothing and deletes +nothing; two rotations racing serialize on the key row; the same key set again bumps +nothing. A rotation is rare, and the alternative, +rewriting every object in place while pushes, restarts, storage switches and further +rotations race the rewrite, is where the complexity would be; with this, nothing but the +current key ever reads an object. The +server builds every key from ids it validated +(`[A-Za-z0-9_-]{1,64}`) and the caller's own email; the client never names a key, and the +workspace storage permission rules are not consulted (the same stance as volumes). Only an +unscoped user token may reach the routes: a job token can carry an `on_behalf_of` identity and +every scoped token (guest, embed, app policy, MCP) was minted for something narrower. + +The backup is keyed by the email like the browser's own stores are (`userScopedDb` scopes +IndexedDB by it): a user whose email changes starts from an empty history on both sides, and +the objects under the old hash stay in the bucket unread. Carrying them over would need a +server-side re-key (decrypt with the old suffix, encrypt with the new, move every object) in the +email-change flow, which this design leaves out. + +An image is accepted only as a base64 data URL of at most 4 MB and stored verbatim, so it +serializes back into a pull answer at its stored size; anything JSON would escape could grow +several times and defeat the pull budget. + +`push` carries `owner`, the email the browser prepared the batch for, and the server refuses a +mismatch with 409: an in-place account switch must not file one user's sessions under another's +prefix. The client captures its user at flush start and checks every store handle's name +against it for the same reason. + +The feature is on wherever the workspace has primary storage, and off with +`ai_config.sessions_storage_disabled` (the `copilot_disabled` pattern: no migration, carried by +settings export and the CLI). A build without `parquet` has no routes (404), a workspace without +storage and nothing to stand in for it answers `enabled: false`; either turns the backup off +for ten minutes, after which the +page asks again on its own (a flush for whatever is pending, and a restore), and the AI +settings page tells the mirror at once when the switch is saved there (the off state is +forgotten, the rows that went stale are marked again, a restore runs). + +## The instance store standing in + +A workspace without storage of its own keeps its backups in the instance object store +(`object_store_cache_config`, loaded the way every other use of it is, so never with +`DISABLE_S3_STORE`; the plan is checked on every request and Pro never falls back, since a +store loaded before a switch to Pro stays loaded), under the same layout and the same +per-user key, +while the instance setting `ai_sessions_instance_storage_fallback` allows it (on unless set +to false; the instance settings page shows it under Object Storage). A build without +`private` has neither workspace storage nor the quota below, and never falls back. Every +answer says which kind of store it came from (`fallback`), and the instance store is named +(`storage_id`) by what locates its objects, the endpoint, region and bucket its settings +resolve to, in a namespace of its own: moving the instance store to another endpoint under +the same bucket name is a storage switch for the browsers, and a workspace bucket is never +taken for it. The location is kept with the loaded store, so a server whose reload is still +pending names the store it writes to. A route decides between the workspace's storage and +the instance store from the row it reads the generation from, so a push lands in the +instance store only under a generation read while the workspace had no storage. The store a +workspace's backups live in is resolved in one place +(`ai_session_backups::workspace_store`), for the routes and for the rotation's deletion of +older generations alike. + +Configuring a storage for a workspace that had none (`edit_large_file_storage_config`) bumps +the backup generation in the transaction that sets it, so everything the workspace left in +any instance store sits under a generation the routes never read again: a later return to +the instance store, whichever it is by then, starts from a newer one. Every storage settings +change then deletes from the instance store, off the request, the workspace's generations +older than the one it committed, whether the setting is on or off (copies from when it was +on may be there). Nothing live is older, whatever happens next: a deletion that is slow, cut +short, or overtaken by the storage being dropped or pointed at the instance store's own +bucket touches only generations nothing reads. Dropping the storage bumps nothing and is a +switch like any other: the rows go stale, the sessions are pushed whole into the instance +store, and the old bucket keeps its copy. On the browser side a removal owed to an instance +store (the row names it apart, `storageName` in `sessionMirrorPlan.ts`) is retired by any +answer from the workspace's own storage, since that storage being there means the +generation moved past the copy; one owed to a workspace storage still waits for that +storage, whatever the instance store answered. Copies a deletion missed stay in the +operator's bucket unread, as a deleted workspace's copies do. + +On CE the bytes in the instance store count toward the workspace's storage quota under a +storage name of their own (`_ai_sessions_fallback_`, listed by the periodic recount while +the workspace has no storage of its own, and left out when there are none), so a member +cannot fill the operator's bucket past what the workspace may use; on EE, where workspace +storage has no quota either, nothing bounds them but the per-push caps and the instance +setting. + +## Conflicts and deletion + +Last write wins across devices. The head carries no manifest; `pull` lists the session's prefix +instead, so a stale device that renames or archives a session rewrites only the head and cannot +hide chats a newer device wrote. Two devices continuing the same chat still collide. + +Restore brings back only sessions the browser does not have (`importSessions` is write-if-absent, +and skips ids the user deleted in this page) and never overwrites or deletes a local one from +remote state. It covers the workspace and its forks together, and only once every one of them +that keeps backups has listed (a listing that failed leaves the family for the next page load +or workspace switch, or the copy that did list could be the stale one): a session listed by +two of them (moved between them, the old copy not yet removed, since that mark is the moving +browser's, which may never come back) is brought back from the copy that moved last (`epoch`, +the record's move count, which names the marker), the storage's own modification time +deciding between two of the same count, and not from the other, which would otherwise take +the id first and keep the later copy out for good. A workspace's records land together once +its pulls are done, and just before they do the whole family is listed again (members whose +backups were off included, since a move from another device can land in a workspace between +the first listings and the pulls; a family of one, with nowhere else for a copy to show up, is +not): a session a later copy of which showed up elsewhere is left, with the family, for the +next time. Only a user-initiated `deleteSession`, and the retention sweep below, remove the +backup; the next push from +another device that still has the session is refused with `needs_whole` (nothing of it is +written), its row goes stale without a backoff, and that device's next flush sends the session +whole; the workspace-lifecycle +removals (`reconcileSessionsLifecycle`, `deleteSessionsForWorkspace`) leave it, so a session +dropped by a wrong reconcile comes back on the next restore. Objects of deleted workspaces stay +in the bucket. A session moved to another workspace is pushed whole into the new one, and once +that push has landed the copy in the old one gets a removal mark of its own, naming the storages holding +that copy (the row that knew is the new workspace's by then), retried independently until +each of them has answered, even when the old workspace's backups are off at the time (they +may hold the copy still). Filing the removal only after the new copy is acknowledged keeps the +session backed up somewhere at every point. + +A restore writes a session's artifacts and chats before its record, and records nothing for a +session whose pieces could not be written: recording it would let the next flush push the +half-empty local state over the backup. + +Every answer names the storage it came from (`storage_id`, a hash of what locates the objects, +endpoint, region and bucket, not the credentials, which rotate; the instance store standing +in for a workspace without one is named apart, see above) and the backup generation a +key rotation bumps (`backup_generation`). A sync row records both, and a row naming another +storage or generation goes stale and its session is marked again: a workspace pointed at a +new bucket, or whose key was rotated, holds nothing, and the server looks nowhere else, so +the next flush carries the session whole. A switch leaves the old copy where it was, so the +row rewritten under the new storage records the old one (`alsoIn`, one entry per storage +the workspace was on), and a removal is done only once every storage holding a copy answered +it, whatever the generation (a rotation deleted the older generation's copy anyway): each +answer narrows the row to the storages still holding one, and the mark waits for them to +answer, so a switch back never brings a deleted session back. That includes the rows a flush has just written, when a later answer of the +same flush names another storage or the session was pushed in part on top of a row from the +old one; a session whose own parts were answered from different storages is not settled at +all. The listing a restore starts with runs the same check, so a storage switch is noticed at +the first push after it or on the next page load, whichever comes first. + +## Retention + +`ai_config.sessions_retention_days` (per workspace, in the AI settings; unset by default; +the `sessions_storage_disabled` pattern: no migration, carried by settings export and the +CLI; 1 to 3650) puts an age on sessions, counted from their last activity. Each side applies +it with its own clock against its own timestamps, so no clock is compared with another +machine's, and the two do not time the same event: the server counts the last push that +completed, a browser its last local activity, which includes reading new messages and is not +pushed. A backup swept while a browser still reads its copy comes back once that browser +writes to the session again (its incremental push is refused and goes whole): + +- The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor + about every 40 minutes on each server, one pass at a time under a session-level advisory + lock). For every workspace with a retention it takes the store its backups live in, its + own storage or the instance store standing in, decided from the row it reads the + generation from as the routes do, names the users under the generation prefix + (`list_with_delimiter`) and lists each user's `index/` once: one object + per session, nothing of what the sessions hold. A session whose marker is older than the + retention is removed under its lock (`lock_session`), once its markers, listed again + there, are still all older: a push that renewed the session between the walk and the lock + keeps it, and one split over parts either holds the lock or has the session unlisted with + its token next to the markers (`index/{sid}/push`), which the sweep leaves alone while the + token is younger than the retention: an older one is a push a browser abandoned, whose + landed parts nothing lists, and it goes the same way. Before deleting anything the sweep + writes a record next to + the markers (`index/{sid}/sweep`, not an epoch, so neither `list` nor `pull` counts it), + and `remove_session` deletes it last: a removal cut short, its markers already gone, is + found by the next pass and finished, unless a push listed the session again first. At + most 1000 sessions per workspace and pass; the rest wait for the next. `list` leaves an + expired marker out of its answer meanwhile, so a browser never restores a session the + sweep has not reached. The marker's modification time is the storage's clock and the + cutoff the server's. The sweep reaches only the backups the routes would: a deleted + workspace's stay in its storage, and so do those a workspace keeps in the instance store + once `ai_sessions_instance_storage_fallback` is set to false. +- The browser sweeps its own stores when a tab resolves the logged-in user + (`sweepExpiredSessions`, from the one `onUserChange` in `sessionState.svelte.ts`), before + that tab reads a single session. A session whose last activity is older than the retention + by the browser's clock is deleted locally, record, chats, images, attached files and + artifacts. A restored session carries the backup's time as its last activity, the storage's + clock, so it counts from the later of that and the moment it was restored here + (`restoredAt`): a browser clock ahead of the storage's never deletes a session it just + brought back. Archived sessions count like any other, and persisted unsent drafts by their + pending workspace. + + The stores are shared by the user's tabs, and each keeps copies of the sessions in memory, + so every tab holds a shared Web Lock from before it reads them until it stops using them, + and the sweep deletes only while holding that lock exclusively, requested if available: + granted exactly when no tab of the user has the sessions loaded, which is why the sweep + runs where it does and nowhere else. Nothing holds a copy of what it deletes and nothing + writes the stores meanwhile, so it deletes one record at a time and without re-reading. It + also takes the tab lock the flush and the restore take, again only if available, so neither + plans nor stages a session half deleted; like the restore, it does not run where Web Locks + do not exist. With several tabs open nothing is swept, until one of them reloads alone. + + The hold is only as good as the tabs that take it, so a tab still running a build from before + it has the sessions loaded and holds nothing. A tab loaded after that one, across a deploy, + can sweep a session the older tab has in memory, and a write there afterwards brings the + record back without its chats, which the next flush pushes. It needs a tab left open across a + deploy, a session untouched for the whole retention, and the user going back to that session + in the older tab; the next sweep deletes it again. The same window is open to the + workspace-lifecycle delete in `reconcileSessionsLifecycle`, which no lock guards at all. + + What deletes is the retention the server gives as the sweep runs, asked for under both locks + (`POST /workspaces/session_workspace_retention`, its own route rather than a field on the + lifecycle status, whose answer a tab loaded before this version still reads). Never a + remembered one: a retention raised or cleared since would otherwise delete a session that is + now within it, and a persisted unsent draft has no backup to come back from. What the sweep + keeps in localStorage decides only whether to ask again — it asks when it has asked nothing + yet, when the answer it has is a day old, or when that answer marks a session expired — so + an ordinary load costs no request at all. An answer that does not arrive within five seconds + leaves the sessions for the next load rather than delete on what this browser guessed. That + route answers for a workspace the caller can be authed into, unlike the status: a status is + what to do with the caller's own sessions, a setting is the workspace's to tell, so a + disabled membership is told nothing though its sessions still reconcile. + + Each session's record goes before its pieces, so nothing plans a push for it afterwards, + and a localStorage key written before the record and removed once every piece is gone makes + a later sweep finish a deletion that failed, unless a restore brought the session back + since. The record is deleted without the tombstone a user delete leaves, which is what lets + a restore bring it back. The session's dirty mark and sync row go with it (`sessionSwept`), + unless the row still carries a removal or a restore's staging. Nothing is sent to the storage: the local + copy's age says nothing about another device's, which may have pushed the session since, + and the server applies the rule to the backup on its own. A session swept here that the + storage still lists comes back on the next restore. + +## Limits + +Push bodies are packed to about 8 MB (UTF-8 bytes as sent), at most 100 entries, 200 removals and +4000 pieces each (the server's caps, with 32 MB on the body, and 100 chats, 500 images or 1000 +deletes per entry, since every piece is an object-store call); an entry that +outgrows the target is split into chat-only parts (the artifacts and deletes on the last, the +head on the last too for an incremental push and on the first part, whatever it carries, for a +push of the session whole), and deletes +past the per-entry cap are carried over to the next push, which the session stays marked for. A chat above +16 MB or a session's artifacts above 8 MB are left out with a console warning; a chat that grew +past the cap after it was backed up has its copy deleted, so a restore never presents the old +transcript as the current one. A 413 fails only the sessions of that request. A +request the server refuses (any other 4xx but 404/403/409) stops the backup for the page but keeps +the marks and the sync state, so the next load tries again; a session the server reports it could +not store stays marked and is retried with backoff. A move files the old workspace's removal +mark before recording the new copy's row, so a mark that could not be written leaves the move +to be planned again. A workspace that answers `enabled: false` +marks its sync rows stale (the next push after storage returns carries every session whole, +since a new storage may be a new bucket) and leaves its dirty marks where they are (a move into +it must still remember the old copy); it keeps the removal marks of sessions that had been backed +up, so one deleted while backups are off does not come back once they are on, and drops the +removals of sessions never backed up from this browser, so a storage-less instance does not +collect one mark per deleted session forever. A user delete whose removal mark cannot be written +(localStorage full) is carried by the session's sync row instead (`removed`), which the flush +and the restore read like a mark; a session without a row yet (its first push may be in +flight) gets a row saying only that, and every row write keeps a removal filed meanwhile, so +the push's own row cannot erase it. Pull bodies are +capped at 64 KB. Pull answers up to 20 ids within a 32 MB +budget: a session's size is known from the listings before anything of it is read, one that +would not fit is deferred unless it is the first of the answer, in which case it comes in +pages: the answer carries what fits in key order (at least one object, so every page makes +progress) and names where the next picks up (`next`, a cursor the browser sends back as +`resume` with that session alone). Every page carries a fingerprint of the session's listing +(marker, keys, sizes, modification times, entity tags and versions, since a store reports +modification times coarsely and an object rewritten at the same size within that grain would +otherwise fingerprint the same) taken before anything of it is read, and the server +takes it again once the page is read: a page the backup moved under (a push landing object by +object) is read again, a few times, then answered as `moved`, and the browser starts the +session over on that or on two pages whose fingerprints differ. The browser writes each page's pieces as it arrives, over whatever +an earlier restore cut short had staged (the session is absent locally, so its pieces have no +local edits to keep, and the backup may have moved on), and the record, which is what makes +the session visible, only with the last page. A restore in progress keeps a staging row for +the session (the ids of every chat, image, artifact and version it wrote), which outlives it +if it is cut short; the next restore deletes the staged pieces the backup no longer has, by id +and never by clock, before the record lands (once the record is there no restore looks at the +session again, and a flush would push them back), and a prune that could not run leaves the +session, its pieces and its staging row for the restore after. A restore holds the user's tab +lock while it runs, so two tabs cannot each write the same absent session's pieces over the +other's, and runs only where Web Locks exist (a secure context: https, or localhost); on a plain +http origin the browser still backs up, and its sessions come back on a secure one. A +restore never writes an older record over a newer +one; between pages it holds nothing but the sync +row being assembled, whose chats also admit the images of a later page. Every page carries a +fingerprint of the session's listing (`listing`), taken before anything of the page is +listed or read, so an object landing after it is in the next page's; a session whose +fingerprint moved between two of its pages (a chat added by another device could sort before +the cursor and be missed) starts over, up to three times, then waits for the next restore. An object that grew +since the listing (a push replaced it) ends its page just before it and the answer names that +spot, so the next page sizes it anew rather than the session being imported without it. A pull sees every key of a session's listing but keeps the 5000 smallest +past its cursor (a page is defined by key order, and the store promises none), so a session +grown without bound by valid pushes cannot grow the answer's memory through its metadata +either; removing a prefix and a rotation's deletion stream their listings. `list` scans at most 50 000 index markers, keeps the newest 500 as it goes and answers with +them (`truncated` says when there were more); the restore takes 50 of them. Every read checks the object's size before buffering it: one larger than any push writes +(32 MB) is planted, whatever its listing said, and skipped, since whoever holds the bucket's +credentials can put anything at a predictable key; one larger than its listing said grew +since (a push replaced it) and ends its page, for the next pull to size anew. A dirty mark that cannot be written +(localStorage full) records its bump on the session's sync row instead (`extraV`, counted +with the mark's counter and kept by every row write, so a push in flight cannot retire it); +a session without a row yet (its first push in flight) keeps the bump in the page, and the +next row write takes it onto the row in the same transaction, so the row the push writes +cannot retire the mark with the bump unseen; every load's backfill marks again a session +without a row, with a stale one, or with one carrying bumps no push has covered. +A mark or removal for another user (a write that landed after a switch) reaches that user's +rows through a connection of its own, since the shared handle follows the current user. Nothing is read past the budget, whatever a session holds. A +restore takes the newest 50 sessions per workspace: every visible session gets a runtime, and +each runtime's history load reads the whole chat store. On CE the push checks the storage quota +and bumps usage by bytes written (an over-count on overwrites; the periodic recount settles it). diff --git a/docs/app-policy-email-removal.md b/docs/app-policy-email-removal.md new file mode 100644 index 0000000000..fb4cf100fd --- /dev/null +++ b/docs/app-policy-email-removal.md @@ -0,0 +1,47 @@ +# Removing `policy.on_behalf_of_email` + +An app's identity is `policy.on_behalf_of`; the address beside it is a function of that +principal. `on_behalf_of_email` is no longer required — a policy carrying only a principal +executes, deriving the address — but it is still written on every save, and that is the only +thing holding it in place. + +## The gate + +`get_on_behalf_of` gained its derive-when-absent fallback in **1.810**. Every replica before that +*requires* the key and errors outright without it, so it would 400 every anonymous, publisher and +guest app saved by a newer one. A rolling deploy runs both versions at once, which is why the +write stays until no replica older than 1.810 can be live — in practice, once +`MIN_KEEP_ALIVE_VERSION` (`windmill-common/src/min_version.rs`) has passed it. + +There is no `MIN_VERSION_*` constant for this and it does not need one: those exist to gate +behavior at runtime or to trip the build when a constraint expires, and nothing here does either. +The key is written unconditionally, so no replica ever meets its absence until someone follows +the steps below. + +## Step 1 — stop writing the key + +- `stored_on_behalf_of_email` in `windmill-api/src/apps.rs`, and the `create_app` / + `update_app_internal` call sites that store what it returns. +- The CLI and frontend workspace-deploy paths (`cli/windmill-utils-internal/src/deploy.ts`, + `frontend/src/lib/utils_workspace_deploy.ts`). These send the address *instead of* a principal + for a cross-workspace deploy, which is the one case where it is the only identity available — + so this is "stop sending it once the target resolves a principal itself", not a deletion. + +Policies written before this keep their key and keep being read from it; they agree with their +principal, so nothing has to strip them. + +## Step 2 — drop the field + +Remove `on_behalf_of_email` from `Policy` and the fallback in `get_on_behalf_of`, which then +always derives. Optionally strip the key from stored policies. + +This can ship with step 1. It is written separately because step 1 alone is revertible without +touching stored data or the response schema, and because the gate above is what makes either +step safe — nothing about step 2 needs its own waiting period. + +## Why the address is not derived on read + +Read paths return the stored address verbatim rather than recomputing it. Deriving on read means +resolving a principal that, for a draft, is caller-controlled — which turns the read into an +oracle for addresses the caller cannot otherwise see, and leaves a principal that resolves to +nobody with no address at all. Both were live defects while the read paths did derive. diff --git a/docs/auth-surface.md b/docs/auth-surface.md new file mode 100644 index 0000000000..21d42dd39b --- /dev/null +++ b/docs/auth-surface.md @@ -0,0 +1,80 @@ +# Auth surface: facts that are easy to get wrong + +Symbols, not line numbers, are cited: they drift less. + +- **Credential precedence** (`windmill-api-auth/src/auth.rs` `extract_token`): `Authorization: Bearer` + → `token` cookie → `?token=` query param. A URL with `?token=` is a credential on every route, but + an existing cookie silently wins over it. +- **`AUTH_CACHE`** caches a token's identity for 120 s. Deleting a token row does not purge it: the + DB trigger (`migrations/20260316000001_token_hash_pk_swap.up.sql`) notifies only for + `label = 'session'` rows, and `delete_token` never calls `invalidate_token_from_cache`. +- **Sessions** are `token` rows with `label='session'` plus the HttpOnly `token` cookie, minted only + by `create_session_token` (`windmill-api-users/src/users.rs`). `GET /api/users/refresh_token` + mints one for any non-job token but returns plain text, no redirect. +- **`tokens/impersonate`** (superadmin) returns a multi-use token and sets no cookie. +- **`max_token_expiration_days`** caps `POST /users/tokens/create` and `tokens/impersonate`, by + shortening the stored expiration (`cap_token_expiration`), never by refusing: the CLI + authorization page, `wmill user create-token` and the editor's language-server token all pick a + lifetime without reading the setting, and CLIs already installed never will. The CLI signs in + again on its own when its token expires, which is why the authorization page labels it + `cli-login:`, reserved in `is_user_token` so its expiry does not email the user. A token + owned by a service account is exempt: one in the workspace the token names, or in any workspace + for a workspace-less token (for `tokens/impersonate`, the impersonated account). Any workspace + admin can therefore create and impersonate a service account to hold an uncapped token, so the + ceiling bounds personal tokens only. Only the stored expiration is capped: the auth lookup never + reads the setting, so tokens that exist when it is turned on or lowered keep theirs, including + none. Deliberately outside it: server-side mints (`create_token_internal` callers such as native + trigger webhook tokens, which never expire for GitHub and Nextcloud), and tokens with their own + fixed lifetime that outlive a short ceiling: sessions (`MAX_SESSION_VALIDITY_SECONDS`, 3 days, and + re-mintable through `GET /users/refresh_token`) and MCP OAuth access tokens (7 days, with a + rotating 30-day refresh token). Any logged-in user can read the setting through `GET + /settings/global/{key}`, which the token form uses to offer only expirations within it. The + settings API and config sync reject any value `parse_max_token_expiration_days` cannot read, since + the token routes would read it as no ceiling; `parseMaxTokenExpirationDays` in the frontend must + accept exactly the same values. +- **A token's label decides whether its expiry raises alerts.** When `delete_expired_items` removes + an expired `token` row, the monitor emails the owner and raises a critical alert (if enabled); + rows registered by `register_token_expiry_notification` also get an "expiring soon" warning first, + except a token whose whole lifetime fits in `TOKEN_EXPIRY_WARNING_DAYS` (7), which gets no row + since the warning would arrive minutes after it was created. Neither happens when `is_user_token` + (`windmill-common/src/auth.rs`) reserves the label, so a token the system mints for itself, + whether from the backend or from the frontend through `tokens/create`, needs a reserved label. An + `ephemeral-` prefix needs no other change (keep it clear of `is_server_minted_label` if minted + through `tokens/create`); a new prefix also goes into the SQL and Svelte mirrors that function's + doc lists. +- **Every superadmin route refuses a job token**: `require_super_admin` + (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs + `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user + token stored as a secret, never `$WM_TOKEN`. Token scopes cannot narrow superadmin routes. +- **`login_type`** (`password` table) is a free-form `VARCHAR(50)`. Password login and password + reset require `login_type = 'password'`; `set_password` also accepts `pending_oauth` and turns + the account into a `password` one in the same statement (an account created ahead of its owner + gets its first credential that way, or through the OAuth claim below). +- **Login links** (`login_link` table, `POST /users/login_links` superadmin-only, + `GET /auth/login_link/{token}` unauthenticated): single-use, ≤15 min, a session cookie and a + 302 to a same-origin `rd`. `require_login_type` on the mint refuses (409) an account whose + `login_type` has moved on — the way a caller re-entering an account it created stops being + able to once the owner has a password or a provider. +- **Pre-approved trial offer** (`cloud_trial_offer`, cloud-only routes under + `/users/cloud_trial_offer`): written by a superadmin at provisioning, consumed by + `{consumed: true}` or by the portal's refusal; `…/go` is the one Windmill→portal hop that + mints a portal login, over the same `CUSTOMER_SERVICE_TOKEN` trust the onboarding hook uses + (`users_ee.rs`, the portal's admin token). It never expires on its own. +- **OAuth login** (`oauth2_ee.rs` `login_externally`, decision in `existing_login_decision`) + matches an existing account by lowercased email only. Same provider → login; a + `pending_oauth` account (see `PENDING_OAUTH_LOGIN_TYPE`) is **claimed** by the first login + whose address the provider itself asserted and did not mark unverified — `login_type` becomes + the client key and the hash is nulled; otherwise `require_preexisting_user_for_oauth` decides: + on, *every* existing account is loggable-into by any provider; off, "exists but with a + different login type". A new account gets `login_type = `. +- **OAuth email trust**: `LoginUserInfo.email_verified` is read leniently (bool or + "true"/"false" strings) and is only consulted for the claim above; only GitHub is filtered to + `primary && verified`; a missing email is fabricated from `name` as `@windmill.dev` and + reaches `login_externally` with `email_asserted = false`. +- **`GET /api/oauth/login/{client}`** is an unauthenticated 302 to the provider — a plain link + from any page starts SSO. +- **`CLOUD_HOSTED`** is presence-tested (`windmill-common/src/worker.rs`): `CLOUD_HOSTED=false` + still enables cloud mode. Of the routes above only the cloud trial offer and onboarding + profile routes are cloud-gated; for the rest, cloud only adds quotas. +- **`CREATE_WORKSPACE_REQUIRE_SUPERADMIN`** defaults to `true` when unset; only the literal + `"true"` enables it when set. diff --git a/docs/clone_repo_and_upload_to_instance_storage.bun.ts b/docs/clone_repo_and_upload_to_instance_storage.bun.ts deleted file mode 100644 index 485ac0c766..0000000000 --- a/docs/clone_repo_and_upload_to_instance_storage.bun.ts +++ /dev/null @@ -1,390 +0,0 @@ -import * as wmillclient from "windmill-client"; -import { basename, join } from "node:path"; -import { existsSync, rmSync } from "fs"; -import process from "process"; -import { spawn } from 'child_process'; -import * as fs_async from 'fs/promises'; -import * as fs from 'node:fs'; - -const UPLOAD_CONCURRENCY = 16; -const CLONE_MARKER_FILE = ".windmill_clone_complete"; - -type GitRepository = { - url: string; - branch: string; - folder: string; - gpg_key: any; - is_github_app: boolean; -}; - -export async function main( - resource_path: string, - workspace: string, - git_ssh_identity?: string[], - commit?: string -) { - let clonedRepoPath: string | undefined; - - try { - console.log("Starting git clone and Blob storage upload process"); - - // Get the git repository resource - const repo_resource: GitRepository = await wmillclient.getResource(resource_path); - - const cwd = process.cwd(); - - if (git_ssh_identity) { - process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity) - } - - // Handle GitHub App authentication if needed - if (repo_resource.is_github_app) { - const token = await get_gh_app_token(); - repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token); - } - - process.env["HOME"] = "."; - process.env.GIT_TERMINAL_PROMPT = "0"; - - // Clone the repository - const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit); - clonedRepoPath = join(cwd, repo_name); - - // Remove .git directory to avoid uploading git history - const gitDir = join(clonedRepoPath, ".git"); - if (existsSync(gitDir)) { - rmSync(gitDir, { recursive: true, force: true }); - console.log("Removed .git directory"); - } - - // Upload to S3 - const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`; - const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace); - - return { - success: true, - message: "Repository cloned and uploaded to S3 successfully", - s3_path: s3Path, - commit_hash: commitHash, - file_count: fileCount, - }; - - } catch (error) { - console.error("Error in git clone and upload:", error); - throw error; - } finally { - // Clean up cloned repository - if (clonedRepoPath && existsSync(clonedRepoPath)) { - rmSync(clonedRepoPath, { recursive: true, force: true }); - console.log("Cleaned up cloned repository"); - } - } -} - -async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise { - const sshIdFiles = await Promise.all( - git_ssh_identity.map(async (varPath, i) => { - const filePath = join(cwd, `./ssh_id_priv_${i}`); - - try { - // Get variable value using windmill - let content = await wmillclient.getVariable(varPath); - content += '\n'; - - // Write file with content - await fs_async.writeFile(filePath, content, { encoding: 'utf8' }); - - // Set file permissions to 0o600 (read/write for owner only) - await fs_async.chmod(filePath, 0o600); - - // Escape single quotes for shell command - const escapedPath = filePath.replace(/'/g, "'\\''"); - return ` -i '${escapedPath}'`; - } catch (error) { - console.error( - `Variable ${varPath} not found for git ssh identity: ${error}` - ); - return ''; - } - }) - ); - - const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`; - return gitSshCmd; -} - -async function git_clone( - cwd: string, - repo_resource: GitRepository, - commit?: string, -): Promise<{ repo_name: string; commitHash: string }> { - if (commit) { - return git_clone_at_commit(cwd, repo_resource, commit); - } else { - return git_clone_at_latest(cwd, repo_resource); - } -} - -async function git_clone_at_commit( - cwd: string, - repo_resource: GitRepository, - commit: string, -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const repoPath = join(cwd, repo_name); - await fs_async.mkdir(repoPath, { recursive: true }); - - process.chdir(repoPath); - - let args = ['init', '--quiet'] - if (branch) { - args.push(`--initial-branch=${branch}`) - } - await runCommand(undefined, 'git', ...args); - - await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url); - - await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit); - - await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD'); - - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function git_clone_at_latest( - cwd: string, - repo_resource: GitRepository -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - // Handle Azure DevOps token if needed - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const args = ["clone", "--quiet", "--depth", "1"]; - if (subfolder !== "") args.push("--sparse"); - if (branch !== "") args.push("--branch", branch); - args.push(repo_url, repo_name); - - await runCommand(-1, "git", ...args); - - const fullPath = join(cwd, repo_name); - process.chdir(fullPath); - - if (subfolder !== "") { - await runCommand(undefined, "git", "sparse-checkout", "add", subfolder); - const subfolderPath = join(fullPath, subfolder); - - if (!existsSync(subfolderPath)) { - throw new Error(`Subfolder ${subfolder} does not exist.`); - } - - process.chdir(subfolderPath); - } - - // Get the commit hash - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function uploadDirectoryToS3( - directoryPath: string, - s3BasePath: string, - workspace: string, -): Promise { - console.log(`Uploading ${directoryPath} -> ${s3BasePath}`); - - // Walk once into a flat task list so we can drive a bounded-concurrency pool. - const tasks: { localPath: string; s3Key: string }[] = []; - function walk(dir: string, s3Path: string) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name); - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - walk(fullPath, s3Key); - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }); - } - } - } - walk(directoryPath, s3BasePath); - console.log(`Discovered ${tasks.length} files to upload`); - - let nextIndex = 0; - let uploaded = 0; - let lastReport = 0; - async function worker() { - while (true) { - const idx = nextIndex++; - if (idx >= tasks.length) return; - const { localPath, s3Key } = tasks[idx]; - const fileContent = fs.readFileSync(localPath); - const blob = new Blob([fileContent], { type: 'application/octet-stream' }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: s3Key, - requestBody: blob, - }); - uploaded++; - if (uploaded - lastReport >= 25 || uploaded === tasks.length) { - lastReport = uploaded; - console.log(`Uploaded ${uploaded} / ${tasks.length} files`); - } - } - } - await Promise.all( - Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker()) - ); - - // Marker is the LAST write — its presence is what the viewer checks for. - const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`; - const markerBody = JSON.stringify({ - completed_at: new Date().toISOString(), - file_count: tasks.length, - }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: markerKey, - requestBody: new Blob([markerBody], { type: 'application/json' }), - }); - console.log(`Wrote completion marker: ${markerKey}`); - - return tasks.length; -} - -function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise { - const nargs = secret_position != undefined ? args.slice() : args; - if (secret_position && secret_position < 0) - secret_position = nargs.length - 1 + secret_position; - - let secret: string | undefined = undefined; - if (secret_position != undefined) { - nargs[secret_position] = "***"; - secret = args[secret_position]; - } - console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`); - - return new Promise((resolve, reject) => { - const process = spawn(cmd, args); - - let stdout = ''; - let stderr = ''; - - process.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - process.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - process.on('error', (error) => { - let errorString = error.toString(); - if (secret) errorString = errorString.replace(secret, "***"); - console.log(`Shell command FAILED: ${cmd}`, errorString); - const e = new Error( - `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}` - ); - reject(e); - }); - - process.on('close', (code) => { - if (stdout.length > 0) { - console.log("Shell stdout:", stdout); - } - if (stderr.length > 0) { - console.log("Shell stderr:", stderr); - } - if (code === 0) { - console.log(`Shell command completed successfully: ${cmd}`); - resolve(stdout); - } else { - reject(new Error(`Command failed with code ${code}: ${stderr}`)); - } - }); - }); -} - -async function get_gh_app_token() { - const workspace = process.env["WM_WORKSPACE"]; - const jobToken = process.env["WM_TOKEN"]; - const baseUrl = - process.env["BASE_INTERNAL_URL"] ?? - process.env["BASE_URL"] ?? - "http://localhost:8000"; - const url = `${baseUrl}/api/w/${workspace}/github_app/token`; - - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${jobToken}`, - }, - body: JSON.stringify({ job_token: jobToken }), - }); - - if (!response.ok) { - const errorBody = await response.text().catch(() => ""); - throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`); - } - const data = await response.json(); - return data.token; -} - -function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { - const url = new URL(gitHubUrl); - return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`; -} diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index dc06a22fa0..6f7dc5871e 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -13,8 +13,8 @@ the dominant way dbt is orchestrated today. - **In**: run an unmodified dbt project synced into Windmill, one Windmill job per invocation, live per-model observability, dbt models as first-class assets in the existing asset graph. -- **Out**: one Windmill job per dbt model, `state:modified` / slim CI, - `dbt docs` hosting, semantic layer, dbt platform integration. +- **Out**: one Windmill job per dbt model, slim CI orchestration, `dbt docs` + hosting, semantic layer, dbt platform integration. - **CE**: the runtime, the manifest ingest, the asset graph and every piece of UI ship in CE, as do all adapters except two. Only the `mssql` and `oracle` adapters are EE, mirroring the native `ScriptLang` boundary (decision 21). @@ -35,8 +35,8 @@ the dominant way dbt is orchestrated today. | 10 | Private repo auth | Not applicable: the project is synced, not fetched | | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | -| 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below | -| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | +| 13 | Manifest storage | Sidecar table for nodes/edges; the whole manifest is kept once per environment, for deferral — see below | +| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** and real column schemas come from the engine's parquet index, opt-in per project — see below | | 15 | Node rendering | Asset nodes per model plus one runnable node for the script | | 16 | Progress | Live, from the JSON event stream | | 17 | Test failures | Honor dbt's own `severity` | @@ -47,6 +47,8 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | +| 26 | Deferral | A durable state per environment, published by the runs whose relations are the script's; `defer` is a per-run toggle. See below | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +159,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -287,6 +293,32 @@ dbt hands the driver — it is written beside `profiles.yml` and pointed at by `sslrootcert`, as it is for a translated postgres resource. `profile.schema` and `threads` from the descriptor override their block keys rather than joining them. +**A `snowflake_oauth` warehouse behaves like a key-pair `snowflake` one**, +although its credential is an access token that lasts ten minutes: + +- The token goes under the `authenticator` each engine reads as an access + token: `oauth` on dbt-core 1.x, `jwt` on the Rust engines. Those read `oauth` + as the refresh-token flow and refuse a profile without client credentials, + while dbt-snowflake has `jwt` only from 1.9 and the 1.x engine can resolve 1.8. +- Every dbt process that logs in re-resolves the warehouse first + (`PreparedProject::refresh_profile`), and resolving an expired OAuth token + refreshes it. So the build, the `after_all` tests, a node retry and the + column-lineage pass each start with a live token, unless resolving fails: the + process then keeps the profile it already has, token included. +- Run identity masks credentials (`RenderedProfile::identity`), so a token + refreshed between a failure and its retry still matches. +- The OAuth connect flow asks for `database`, `warehouse`, `role` and `schema` + (`resource_fields` in `oauth_connect.json`). No token response carries them, + and dbt needs a database. + +One gap stays: a login after the token its dbt process started with has expired +fails. That is a thread whose first connection opens late in a long process, or +a process's first login when the token it was handed had only seconds left. +Refreshing tokens ahead of expiry would narrow it, but a token's lifetime is not +stored, so no margin fits every provider. Closing it would mean handing dbt the +refresh token and the instance's client secret, which any model can read on +dbt-core 1.x. + Three things follow, and they are the reason for the rule rather than consequences to work around. @@ -429,7 +461,9 @@ Two things make that safe rather than a widening: model set and its relations are already visible to them. - **`raw_code` is gated separately**, on an `EXISTS` against `script` in the authed transaction. The body of a model is the project's source code and stays - behind access to the project, whatever the shape query resolved. + behind access to the project, whatever the shape query resolved. `column_schema` + and the column trace behind `/jobs/dbt_column_lineage/{id}` take the same gate, + for the same reason: both are the shape of what the author wrote. The path and hash coming from the job row rather than the query also means a caller cannot pin one project's version while naming another's run. @@ -648,10 +682,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +700,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +801,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -857,6 +971,8 @@ profile: select: ["tag:nightly+"] exclude: [] test_behavior: build # build | after_all | none +column_lineage: false # opt in to the static-analysis pass that + # produces column-level lineage (decision 14) vars: # typed: numbers/bools/lists keep their type, run_date: "{{ run_date }}" # and string leaves take job arguments strict: false @@ -1143,19 +1259,513 @@ block, since that is what `dbt_run_state` saves and `invocation_args` publishes. without failing. Overriding this would make the same project behave differently on Windmill than locally, breaking the core promise. +## Durable state per environment, and what defers to it + +`dbt --defer --state ` resolves a `ref()` the run does not build to the +relation the manifest in `` names, instead of to the schema this run writes +into. That is what lets one model be rebuilt into a scratch schema without +rebuilding everything above it, and it needs a manifest of the environment the +project actually lives in. + +Nothing that already existed could supply one. `dbt_run_state` answers a +different question — it holds the LAST run whatever its outcome, keyed by the +principal, so `dbt retry` can resume its failures — and the worker-local +generations behind it are a cache: the next run of a project usually lands on a +worker holding neither artifact. So the state is its own table, +`dbt_environment_state`, one row per (workspace, script path, environment), +holding `manifest.json` and `run_results.json` from the last SUCCESSFUL run. +Success is half of the contract: a relation a later run defers to has to exist. + +### The environment is the warehouse, the target and where they resolve to + +The workspace warehouse's name, the target dbt actually runs, and the database +and schema that target resolves to — the pair `relation_root` reports to the +graph's drift check. Each component is length-prefixed rather than joined on a +separator — `|||`, each written `:`, +so `main`/`prod`/`analytics`/`dbt_wh_defer` is stored as +`4:main|4:prod|9:analytics|12:dbt_wh_defer`. A target name and a schema are both +the user's own strings, so `prod|analytics` + `scratch` and `prod` + +`analytics|scratch` would otherwise be one key, and a profile moving between them +would read as the same environment rather than as one nothing has published. What +a message names is spelled out instead, never the encoded key. + +The target is the EFFECTIVE one, not the descriptor's `profile.target`: a +descriptor naming none inherits the workspace warehouse's, or the default in the +project's own `profiles.yml`, so reading the descriptor's would file every +inherited target under one empty name — and a `target.name` macro decides where a +model is built. + +The last two are in the key because deferring is resolving a relation NAME. A +warehouse repointed at another database, or a `profile.schema` moved by a +redeploy, keeps the first two while putting every relation somewhere else, and a +manifest is a list of relation names — there is no other way to notice. Keyed on +the first two alone, such a move would hand the next deferring run the names of +relations that are no longer there. Keyed on all four, it reads as an +environment nothing has published yet, which is what it is. + +What the key deliberately does NOT carry is the resolved connection. That is the +`profile_digest` a retry is held to. It masks credentials, but it still moves +with changes that move no relation, like another Snowflake warehouse or role; a +warehouse pointing somewhere else entirely is decision 11's accepted limitation, +spelled the same way here as everywhere else. + +Today one script has one environment, because a descriptor fixes both the +warehouse and the target and a run cannot override either. The key is what makes +the *later* item — fork and preview environments — an addition rather than a +migration, and what makes a profile move detectable now. + +### Which runs publish it + +A successful `build` that did not itself defer, and whose graph becomes what the +script owns (`GraphRefresh::publishes_ownership`) — the same condition as the +graph's and the same reason: an invocation that scoped its own model set — a +`vars` or `select` override, or a descriptor dynamic by construction — describes +where the CALLER put those relations, not where this project's models live. +Publishing it would point every later deferral at one caller's scratch schema. + +**A run that deferred never publishes, whatever narrowed it**, and that is a +separate condition rather than a consequence of the first. A deferring run built +some of the relations its manifest names and resolved the rest out of the state +it read, so recording that manifest would claim relations nothing built — and a +model renamed since would be recorded under a name only a full build creates, +breaking every later deferral until one repairs it. `publishes_ownership` cannot +see this: it reads the caller's overrides, and a descriptor that already narrows +`select` needs none. + +A `retry` publishes nothing. Its `run_results.json` names only the nodes it +redid, so the environment would come to claim a run of a handful of models. The +environment's state is therefore the last full successful build, exactly as dbt +Cloud's "last successful run" is, and a run recovered by a retry leaves it at +the previous one. + +The AUTOMATIC in-job node retry is the same artifact under a different name: a +build it recovers is a successful build, but the `run_results.json` on disk is +the retry's. Such a run publishes the manifest **without** results, rather than +with a set describing some other slice of the build — the manifest is a function +of the project rather than of what ran, so deferral is unaffected. A `result:` +selector is the one thing left with nothing to read, and it is refused by name +against such a publication rather than passed to dbt (see "Selectors that read +the state" below). + +Under `test_behavior: after_all` the stored `run_results.json` is the test +phase's, because that is what the second invocation leaves in the target +directory — the same artifact a local `dbt run && dbt test` leaves behind. + +**What that condition means for what the artifacts may carry**, and why this +table is keyed by environment where `dbt_run_state` is keyed by principal. dbt +records the invocation's flags into `run_results.json`, and Windmill resolves +`$var:` / `$res:` references before dbt sees them — which is exactly why the +retry state is per-principal, so one caller's resolved `select` and `vars` are +not restorable by the next. Here they cannot be one caller's: a publishing run +added nothing of its own, and a descriptor that interpolates a `{{ }}` +placeholder into `vars` never publishes at all, so what is recorded is the +descriptor's own arguments — the script's content, which anyone entitled to run +it may already read. Widen the publish condition and that stops being true. + +### Where the blob goes + +`run_results.json` is small; `manifest.json` is not, and grows with the project +(535 KB on a two-model fixture). Each takes the same two homes: inline in the row +under `DBT_STATE_INLINE_MAX_BYTES` (8 MiB), and the INSTANCE's object storage +above it, with the row keeping the key. Inline is what makes the feature work on +an instance that has configured no storage at all; the ceiling is what stops one +project's manifest from becoming a multi-megabyte row rewritten by every run. A +project past the ceiling with no storage configured is told so, in the job log, +naming the setting and the variable — the run itself still succeeds, since +losing the state costs the next deferral rather than the build that just ran. + +**The instance store, not the workspace's**, which is where every other internal +worker artifact already lives (bun bundles, python wheels, job logs, the global +cache). The workspace bucket is the one members read and write through +`job_helpers/*` and `wmill.write_s3_file` with a caller-supplied key, and only +`volumes/` is reserved there — so a manifest under it is one any member could +replace, and the next deferring run would hand dbt an attacker-chosen +`defer_relation` for every unbuilt `ref()` while holding the script's warehouse +credentials. Its compiled SQL would be readable there too, for a project the +reader may have no access to. The consequence to know: a project past the ceiling +needs the instance store configured, which is an EE feature, so on CE the ceiling +is the limit and `DBT_STATE_INLINE_MAX_BYTES` is how it moves. + +Each publication writes its OWN keys +(`wmill_dbt_state///./`) +and the row switches to them in one statement, so an upload never overwrites an +artifact the committed row still names: a run that fails between its two uploads, +or between them and its row, leaves the state pointing at the pair it already +had. The objects the commit displaced are dropped afterwards, never before, since +a reader that has already read the row is about to fetch them; a reader that +loses that race re-reads for as long as the row keeps MOVING, rather than +reporting a state that is there. A reader takes no lock, so successive +publications can each overtake one; an unmoved row whose objects are gone is the +error that means what it says, and a bound on the re-reads is the other, for a +project republishing faster than a run can read. What a publication uploaded and then could not commit is dropped on the +way out — except after a commit that REPORTED an error, where what was lost may +be only the acknowledgement: dropping then would leave a committed row naming +objects that are gone, so an orphan is the cheaper side to take. + +The path and the environment are only a prefix of that key. The row is what says +where an artifact is, which is why state can travel with a renamed script and go +on naming objects under the old path's digest. The rest of the key is the job and +a per-EXECUTION nonce — zombie recovery re-runs a job under its own id, so keyed +on that alone a second attempt would overwrite the objects the first attempt's +committed row still names, then read those keys back as displaced and drop them. + +Publishers of one environment serialize on `pg_advisory_xact_lock`, so only one +of them settles the row and the objects it displaces at a time — an advisory lock +rather than the row's, because the first publish of an environment has no row to +lock and is exactly when two runs of a newly deployed script are most likely to +race. + +### Retention + +None, deliberately, and this is where it differs from the graph tables next +door. Those are pruned by age by the dbt runs themselves because their reader is +a transient run page. This one holds a single row per script per environment, +replaced in place, so it does not grow with runs — and its reader is every later +run of that script, so a project that runs monthly must still find last month's +state. It goes with the script instead: a path no live dbt version occupies any +more clears it, alongside `dbt_run_state` (`clear_dbt_script_state`, +`clear_dbt_script_state_if_path_retired`). + +The write carries a guard of its own, and it names the VERSION rather than the +path: the live dbt script there must be the one this job ran, or a later version +of it (`hash = $n OR $n = ANY(parent_hashes)`). "Some live dbt script is here" — +which is what the retry state settles for — is also satisfied by a script created +at a path this one was renamed away from, and this job's manifest would then +become that project's deferral state. A preview names no version and so publishes +nothing, which is right for a run of content that was never deployed. + +The job's KIND is checked beside it, because a preview carries a caller-supplied +`script_hash` into `runnable_id` (`run_preview_script`): the version alone would +let anyone who may run a job publish arbitrary content as a deployed script's +state. A flow or app step naming a deployed dbt script by path is an ordinary +`script` job carrying that script's own hash, so it publishes like any other run; +only INLINE flow code is a `FlowScript`, and that has no deployed version to +publish for. + +That guard HOLDS the script row (`FOR SHARE`) for the rest of the publication, so +a rename, archive or delete of the path either waits for it or is seen by it. +Read unlocked, it leaves a window where the lifecycle clear finds no row to take, +finishes, and the publication then commits state at a path a new script goes on +to occupy. The script row is taken before the sidecar, which is the order every +other dbt writer takes and what keeps the two off a deadlock. + +An artifact too large for its row is left in the store when the row is cleared, +as a deleted script leaves its bundle: reaching it from the delete would mean an +object-store client in `windmill-common` and a delete that has to land after the +caller's transaction commits, for one object per environment of a script that is +gone. + +### Asking for it + +`defer` is a field on the `build` command block, defaulting to the descriptor's +own `defer:`. A per-run toggle rather than a descriptor-only setting, because the +run that publishes an environment's state and the run that defers to it are two +invocations of ONE script (decision 6: N scripts means N projects): a project +that could only defer by descriptor could never populate the state it reads. + +A project whose profile selects its schema or database with a TEMPLATE — either +delimiter, since dbt renders `{% … %}` blocks as well as `{{ … }}` — is refused a +deferral outright, and publishes no state either: dbt renders those and Windmill +does not, so two renderings resolve to one `relation_root`, and a +deferral after the value changed would resolve every unbuilt `ref()` through the +previous location's manifest. Both sides, because a published template would sit +under a key a literal profile shares, and de-templating later would make that +stale manifest readable as the new location's. It covers a project-owned +`profiles.yml`, a `dbt_profile` resource — one block of the user's own file, +copied through unchanged — and a `profile.schema` written as given. Plainly +absent is different: that is the adapter's default, which does not move. + +A run that asks to defer with nothing published is refused, naming the +environment and the runs that cannot publish one. The alternative — running +without deferral — fails deep inside dbt with a relation-not-found the caller has +no way to connect back to a missing state. An agent worker is refused the same +way and for a reason it can act on: it reaches the database only through the API, +which does not expose this table. + +A `show` defers too, and every engine takes the flags on it. It compiles the +model it previews, so a model whose upstream this environment built and this run +did not is exactly the case a deferral exists for. So does the `dbt ls` that +resolves what a run's selection owns, without which a `result:` selector — which +reads `run_results.json` out of the state directory, and which `select` passes to +dbt verbatim — would fail before the build that would have honoured it. + +The result carries `deferred_to`, the run whose state was used. Without it what +a deferring run built against is unrecoverable, since the next successful run of +that environment replaces the state. + +### Selectors that read the state, and why they are refused rather than passed + +`--state` also feeds dbt's own selector methods, so publishing the state is what +makes `state:modified+`, `state:new` and `result:error+` resolve at all. Only a +deferring run is handed the directory, so a `state:` or `result:` method in +`select` or `exclude` without `defer` is refused before dbt starts. + +Refused, rather than left to dbt, because the engines disagree about it and two +of the three disagree silently. Given a state selector and no `--state`, +dbt-core 1.x raises (`Got a state selector method, but no comparison manifest`, +exit 2), but dbt-sa-cli 2.x and fusion read a MISSING state as an EMPTY one and +exit 0: `state:modified` then selects nothing and the run reports success having +built nothing, while `state:new` selects everything, because against an empty +state every node is new. A scheduled run that quietly stops doing work, or +quietly rebuilds the project, is the failure this state exists to prevent. + +From the DESCRIPTOR they are refused whether or not the run defers, and the +message says so. That selection is also what decides which nodes the script owns, +and the deploy resolves it before any run exists, with no state to compare +against. "Whatever changed last" is not an ownership answer. They describe one +run, so they belong in a run's own `select`. + +`source_status:` is refused under any setting: it compares `sources.json`, which +`dbt source freshness` writes and no run publishes here, so there is nothing to +compare against even while deferring. + +Two more refusals follow from the same argument, that a selector with nothing to +read must say so rather than resolve to a silent answer: + +- A `result:` method while deferring to a state that carries **no** + `run_results.json`. Publishing that is deliberate — a build recovered by + automatic node retry stores the manifest alone, its results describing the + retried nodes rather than the build ("Which runs publish it") — so `defer` + being on is not enough to know the file is there. Answerable only once the + state is loaded, so it is checked right after, naming the run that published. +- Any of them on a `parse`. A parse resolves a selection to store the graph and + never defers, so `defer` would not hand it a state at any setting, and the + remedy the other refusal offers would lead nowhere. It says that instead. + +Matching nothing is then an ordinary outcome for these methods, and for no +others. `state:modified+` selects the empty set exactly when nothing changed +since the published state, which is the answer a CI run wants, so a selection +naming a `state:` or `result:` method may resolve to no nodes. Such a run scoped +its own selection, so what it stores is a snapshot of its own and never what the +script owns, and nothing is un-wired by the empty set. + +The exemption is by METHOD, not by who chose the selection. Exempting every +caller-chosen one would take a misspelled model name, which resolves to nothing +just as surely, and report it as a build that did its work. An ordinary selection +matching nothing stays refused, from a run as from the descriptor — from the +descriptor because that one also decides ownership. + +Only what `select` and `exclude` spell directly. A method reached through a +`selectors.yml` definition is named nowhere the worker reads, and dbt's own +behaviour — including the silent one — is what stands there. + +### `--state` is also a retry's own argument, and that is a trap + +`dbt retry` reads the run it RESUMES from `--state`. Handed the deferral's +directory it resumes the successful run stored there, finds nothing failed, and +reports a green retry having rebuilt nothing — silently, on dbt-core 1.x, which +warns and exits 0. + +dbt-core 1.x has `--defer-state`, the deferral-only half of the pair, so a retry +there passes that and leaves `--state` alone. The Rust engines do not have it, +and a run that deferred is refused a retry on them, before the build: the +alternative is rebuilding the failed nodes with every `ref()` resolving into the +schema this run writes into, which for the narrowed run a deferral exists to +serve means writing them somewhere they do not belong. The automatic in-job node +retry is dropped for the same reason and says so in the log. + +The state directory is passed RELATIVE (`wm_dbt_state`, beside `wm_target` in the +job directory). dbt records the invocation's flags into `run_results.json` and a +later `dbt retry` restores them, so an absolute path would name the job directory +of the run being resumed, which is gone by then. Relative, it resolves against +the project root — whichever job directory the retry landed in. + +Three engine facts found while wiring this up, all worth knowing before filing a +bug against the feature. `dbt retry` on dbt-core 2.x restores **neither** the +resumed invocation's `--vars` nor its deferral: it re-parses with the current +(empty) ones, so a retry of a run that overrode `vars` rebuilds into the +descriptor's schema rather than the run's. That is independent of deferral and +predates it; the refusal above stops the deferring case from being the way it is +discovered. `dbt show` on either Rust engine prints a bare JSON array where +dbt-core frames it as `{"node": …, "show": […]}`, which `run_show` is written +against — so a preview there fails to parse whether or not it defers, and the +deferral itself resolves correctly under it. And neither Rust engine reached +dbt's own service-backed State (`--manage-state`) on any run measured here, so no +flag is passed to disable it. + +Because `select` reaches dbt verbatim, a deferring run also has a `--state` +directory for `result:` selectors, which is why `run_results.json` is stored +beside the manifest rather than the manifest alone. + ## Two decisions the implementation narrowed -**Decision 13 — no S3 copy of the manifest.** The sidecar holds every field the -graph renders; nothing reads a stored `manifest.json`, so writing one to S3 -would be an unread copy of data that is already reproducible by redeploying (or, -for a dynamic descriptor, by the next run). Worth adding the day something needs the -parts the sidecar drops — compiled SQL, macro definitions — and not before. +**Decision 13 — the manifest is stored once per environment, not per version.** +The sidecar holds every field the graph renders, so a copy of `manifest.json` +bought the graph nothing: it is reproducible by redeploying, or for a dynamic +descriptor by the next run. Deferral is the reader that changed that — it +resolves an unbuilt `ref()` through a manifest, and one on worker-local disk +answers for a machine's history rather than for the environment. So exactly one +manifest is kept per (script, environment), replaced by each successful run, +rather than one per version (see "Durable state per environment" above). -**Decision 14 — column lineage is not available.** The decision assumed -`manifest.json` carries column-to-column edges; it does not, in either core -engine. What it does carry is declared column *descriptions*, which are -ingested. Real column lineage would need Fusion (which does static analysis) or -a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. +**Decision 14 — column lineage comes from the parquet index, not the manifest.** +`manifest.json` carries no column-to-column edges, in any engine, and its +`columns` are the ones an author declared in `schema.yml`. Both halves exist in a +different artifact: `dbt compile --static-analysis strict --write-index` writes +`target/index/`, and two of its tables are `dbt.column_lineage.parquet` +(`from_node_unique_id`, `from_column_name`, `to_node_unique_id`, +`to_column_name`, `lineage_kind`) and `dbt.node_columns.parquet` (every column of +every node, with its declared type, its inferred type and its description). + +Three measured properties decide the shape of the ingest. + +**Strict analysis is a stricter dialect.** `select no_such_column from +ref(...)` is `UnresolvedIdentifier (dbt0227)` and exit 1 under `strict`, and +compiles fine under `baseline` (the default). So this is a separate `dbt compile` +with its own `--target-path`, never a flag on the build, and it is opt-in per +project: `column_lineage: true` in the descriptor. Off, nothing changes. On, a +project that cannot be analyzed keeps exactly the graph it had. + +The pass is best-effort about everything that is ITS: a wrong engine, a rejected +analysis, a missing or unreadable artifact, an over-long output and outrunning its +own time budget all degrade to partial lineage or none, plus a line in the job +log saying which. It is not best-effort about the JOB: a cancellation or the job's +own deadline fail it, because swallowing those would let a run that blew its +timeout inside an optional annotation publish a graph and report success. That +split is why the two halves have separate error contracts — the compile owns the +job's semantics and may `Err`; nothing the artifact does or fails to do is a +reason to fail a job, so an absent, unreadable or partial index is a value. The +decode still runs under the job poller, which both heartbeats through it and +ends it if the job is cancelled or completed meanwhile: the job reaching in, not +the artifact reaching out. The +budget is half the job's remaining wall clock, spent on the compile alone, so the +build that follows cannot be starved by it. + +**A failed pass still writes the index**, holding every edge of the models that +did analyze, so the artifact is read whatever the exit status and partial lineage +is a normal outcome. An unreachable *source* is milder still: `RemoteError +(dbt1014)` downgrades that model to `static_analysis: off` and the compile +succeeds. (Strict analysis queries the warehouse catalog for source schemas; a +`ref()`ed model is inferred statically and needs no built table.) + +**The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 — the version +`DBT_CORE_2X_VERSION` pins — accepts `--write-index` and `--write-lineage`, and +its own `views.sql` declares views over both tables, but it writes neither +parquet; only Fusion does today. The ADAPTER decides too: an experimental one +(postgres under `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`) turns static analysis off and +says so only in a warning on an otherwise successful compile. The gate is +therefore "the engine has the flag" (everything but 1.x, whose Python CLI has no +such option) plus "the file appeared", so a later release picking the feature up +needs no change here — and the job log carries the engine's own stderr whenever +no index appears, since without it "no column lineage" has no explanation. + +`lineage_kind` is stored as TEXT, not an enum. Three values exist — `copy` +(passthrough), `mod` (transformed) and `scan` (the column was read to produce the +ROW rather than the value: a join key, a `where` predicate, a `group by`) — and +the engine's own reader maps those three and passes anything else through, so the +set is the engine's to extend. All three are stored, and `copy`/`mod` are kept +first when the bound bites: a `scan` edge reaches every output column of its +model, so it is most of what a project's index holds and would draw as a complete +bipartite graph. Keeping it in the table anyway is what lets a later "show +indirect" view ask for it without every project being redeployed. + +Storage mirrors `dbt_edge` exactly: `dbt_column_edge`, keyed by (path, version, +job) with the same composite foreign key to `script`, so a version's column +lineage dies with the version and a run's snapshot with the sweep. + +**A table of its own, not `dbt_edge.column_lineage` JSONB.** Hanging the links on +the `ref()` edge they sit beneath would inherit its clone, prune, clear and +cascade paths for free, and it does not work: a model reading `{{ this }}` gets +column lineage from itself to itself, and `parent_map` has no self-loop, because +a model does not `ref()` itself. Those pairs have no `dbt_edge` row to attach to. +The loss is not hypothetical — an incremental that selects from `{{ this }}` +(`coalesce(p.dbl, s.dbl)`, `p.up as prev_up`) yields `up → prev_up` with kind +`copy`, a drawn edge meaning "this column carries the previous run's value". +Inventing self-loop `dbt_edge` rows to hold it is not an option either: that +table is `ref()` lineage. The typed column list lands in +`dbt_node.column_schema`, beside `columns` rather than merged into it — +`columns` stays what the author *declared*. + +Two things are user-visible. `column_schema` — every column of a relation, typed +and in the order the model emits them — rides the asset graph the details pane +already fetches, and replaces a panel that could only list the columns an author +happened to document. The edges are served by an endpoint of their own, +`assets/column_lineage`, which the pane asks for the selection it is drawing. + +Both are gated on being able to read the producing project, like the model's SQL: +a column-level view is the shape of what the author wrote, one level finer than +the `ref()` graph, which is ungated only because it draws relations the caller +already sees. A share-link viewer entitled to a dbt run therefore gets its +relations and `ref()` edges, and neither the SQL nor the columns. + +**One request per selection, and the gate re-decided per project.** The endpoint +takes every relation the view has reached and answers their union, because one +selection reaches several — a script's output column can derive from columns of +several models. Holding partial answers between selections instead was tried and +is what a client cache is: it produced a wrong premise for a relation two projects +describe, then staleness on redeploy, then a lost retry. + +The answer is the connected component around those relations, and that component +does not stop at the project that owns them: a relation one project produces is +another's source, so the walk resolves owners, reads their edges, walks, and +repeats for the relations that walk newly reached. Resolving once — for the +relations asked about — stops the trace at the first project boundary. The +security half is that a project reached this way is a project the caller may not +be entitled to, so the scope filter and the project's visibility are re-applied to +every project the expansion discovers, not decided once for the first owner set. + +A PINNED answer is the exception and needs none of it, whether it names a job or +a deployed version: the pin says which stored graph is on screen, and another +project's live graph is not part of it, so it answers for that one project. The +dbt editor pins by version on every selection and the run page pins by job; the +pipeline page pins nothing, and is where the walk crosses projects. + +**The component is bounded, and says when it was cut.** The renderer draws a box +per column, so a component past a few thousand edges is unreadable however it is +served — a synthetic 3000-model project whose models share a column returns 58k +direct edges and 7.3MB. The walk is breadth-first from the asked-for relations and +stops at 5000 edges, so what survives is the part nearest the selection rather +than an arbitrary slice, and `truncated` says so: a trace that stops short is +otherwise indistinguishable from one that ends. + +`truncated` is set only where a project this caller may read is left unread, or +where the walk itself was cut. Neither the expansion's size budget nor a relation +whose owners were never asked about is evidence on its own: a big project's small +component is answered whole, and the relations still waiting to be asked about at +the end are almost always owned by the project already in hand. The order the +rounds run in is what makes that decidable — the owners query comes before the +budget check, so the stop happens with a named unread project rather than a +suspicion of one. + +The FETCH is not bounded the way the answer is: a project's edges arrive whole, +because the walk is what decides which of them are in the component, and a +`LIMIT` would cut a set that need not contain the asked-for relation at all. The +seeds' own projects are therefore read whatever their size — reading them is the +answer. What bounds a single fetch is the ingest's own cap per version; the +budget bounds only the expansion on top of it. + +The walk is in Rust rather than a recursive CTE. `EXPLAIN ANALYZE` on that same +project measured 1243ms against 59ms for the query alone: a CTE has no index to +walk, so the recursive term rescans the doubled edge set once per level (11.7M +rows), while the same walk over a map is microseconds. + +The two halves of a trace — dbt's and the pipeline's — meet at shared node ids. A +DuckDB script's `// column x <- dbt://wh/s/model.col` mints the same +`(dbt, path, column)` node dbt's own lineage does, so the producer graph the asset +graph already carries and the dbt graph are MERGED rather than chosen between, and +a trace crosses that boundary in either direction. What the browser cannot close +in one request is a relation the server discovers whose columns are consumed by a +script that writes into a third project: the producer half of that hop is the +canvas's, not the server's, and the seeds were computed before the answer arrived. + +**The analysis pass takes the build's own `--full-refresh`.** `is_incremental()` +branches on it, so an incremental model reading `{{ this }}` compiles its +self-join — and any `ref()` inside that branch — only when the flag is absent. A +pass that used the descriptor's default while the run overrode it would store +lineage for SQL that run never executed. For the same reason an invocation that +overrides the flag counts as `per_run_models`: its graph is its own, keyed to the +job, rather than standing as the version's. + +That flag is not the whole of it, and the rest is a property rather than a bug to +fix. `is_incremental()` is also false when the target table does not exist, so an +incremental model has **two shapes and one ingest holds one of them**: a deploy +before the first build compiles the cold shape, and the same project deployed +again once its tables exist compiles the incremental one. A static descriptor +re-ingests on neither runs nor time, so what is stored stays whatever the compile +in front of it saw. dbt has no mode that emits both, and re-analyzing per run +would buy a second `dbt compile` on every build to keep a graph nobody asked to +refresh. The contract is therefore the honest one: a version's graph describes +the compile that produced it, and a run that re-ingests describes its own run. ## Concept mapping @@ -1167,7 +1777,9 @@ a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. | `materialized: incremental` | `append` or `merge` (by `unique_key`) | same | | `{% snapshot %}` | `scd2` | same, incl. `_current` handling | | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds | -| declared column metadata | `columns` on the asset node | descriptions only; see the note below | +| declared column metadata | `columns` on the asset node | descriptions only, from the manifest | +| analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in | +| column-to-column lineage | `dbt_column_edge` rows, drawn as a column trace | `dbt.column_lineage.parquet`, opt-in | | model `tags` | node badge | `tag` | | source freshness | `freshness` | `last_success_at` chip | | `run_results.json` | materialization records | `record_materialization` | @@ -1195,7 +1807,10 @@ render through the existing `RunnableNode.svelte` / `AssetNode.svelte` / on the canvas mid-run. `record_materialization` per model. Profile and select pickers in the editor. Per-model failure triage in the run view. -**Phase 4 (not in this PR).** `--defer` and `state:modified`. Partition and +**Phase 4 (not in this PR).** Slim CI: the fork and preview environments a +deferral would name instead of its own. The selectors themselves are here, since +`state:` and `result:` read the published state like any deferral does; what is +missing is a per-branch environment to compare a CI run against. Partition and backfill integration so `BackfillRangeDialog.svelte` works on dbt models. `wmill dbt import ` reading `DbtDag(...)` kwargs. @@ -1215,18 +1830,33 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. +12. **Deferral**: a full run publishes the environment's state; a second run + that builds one downstream model into another schema resolves its unbuilt + `ref()` to the relation the state names, where the same run without `defer` + fails with relation-not-found. +13. **State selectors**: with a state published, `state:modified+` selects + nothing while the project is unchanged and exactly the changed model and its + children after one is edited. Without `defer` it is refused rather than + passed, and a `result:` selector against a state published by a + node-retry-recovered build is refused too, that one carrying no + `run_results.json`. Keep only tests that pin behavior a future change could break. Per AGENTS.md, delete development scaffolding before marking the PR ready. diff --git a/docs/docker-security.md b/docs/docker-security.md index f0a9f23073..5b5196f8b4 100644 --- a/docs/docker-security.md +++ b/docs/docker-security.md @@ -63,3 +63,65 @@ gap, rebuild and republish the `latest` / patch tags: Scan the published images (e.g. Trivy / Defender) after rebuilds to confirm the base-OS finding count stays low. + +# Verifying image signatures, SBOMs and provenance + +Release images are signed and attested at publish time: + +- **cosign keyless signature** on the pushed manifest digest (index and + per-arch manifests), via GitHub OIDC — no long-lived signing key exists + (`.github/actions/sign-attest-image`). +- **SBOMs** are generated at build time (`sbom: true` on the depot build + step) and embedded in the image index as BuildKit attestation manifests — + one SPDX document per platform. They are part of the signed index digest, + so the cosign signature covers them. They are not sent to a transparency + log: SPDX documents for these images run tens of MB, beyond what Rekor or + GitHub attestations accept as payloads. +- **SLSA build provenance** recorded as a GitHub artifact attestation and + pushed to the registry (`actions/attest-build-provenance`). + +## What is covered + +Only images published from a release tag (`v*`) are signed: `windmill`, +`windmill-ee`, `windmill-ee-cuda`, `windmill-slim`, `windmill-ee-slim`, +`windmill-full`, `windmill-ee-full` (`.github/workflows/docker-image.yml`), +`windmill-cli` (`build_cli_image.yml`) and `windmill-extra` +(`publish_extra.yml`). The `:latest` and `:main` tags are repointed on +every `main` push as well as on releases, so they resolve to a signed +digest only until the next `main` build lands — verify a version tag or a +digest, not `:latest`. Development images (`:dev`, branch builds, +`windmill-test`), the dispatch-only RHEL/rpi images and the `caddy-l4` +image are not signed. + +## How to verify + +Signatures are keyless: trust is anchored in the Fulcio certificate identity, +which for these images is the *calling workflow file at a `v*` tag ref* in +this repository. Verify a signature with cosign (v2.x): + +```bash +cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp '^https://github.com/windmill-labs/windmill/\.github/workflows/(docker-image|publish_extra|build_cli_image)\.yml@refs/tags/v' \ + ghcr.io/windmill-labs/windmill: +``` + +Extract the embedded SBOM (per platform; verify the signature first — it +covers the index these documents live in): + +```bash +docker buildx imagetools inspect ghcr.io/windmill-labs/windmill: \ + --format '{{ json .SBOM }}' +``` + +Verify SLSA provenance through GitHub's attestation API: + +```bash +gh attestation verify oci://ghcr.io/windmill-labs/windmill: \ + -R windmill-labs/windmill +``` + +Note for registry housekeeping: cosign stores signatures as extra +`sha256-.sig` tags in the same ghcr package, and the pushed +provenance attestations live there as referrer artifacts — any +tag-retention automation must not prune them. diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 57467e25ec..ae62d46b94 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,11 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 21 registered actions across nine features (`ai_session`, `ai_chat`, -`flow_editor`, `flow_run`, `flow_step`, `trigger`, `command_script`, `hub_script`, -`usage_meter`). Nearly all of the product is uninstrumented, so new user-facing work is the -opportunity to change that. +It currently carries 51 registered actions across nineteen features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, +`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, +`usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the +product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/docs/git-repo-viewer-hub-script.md b/docs/git-repo-viewer-hub-script.md index 7bfbe8160f..a1b5ea1482 100644 --- a/docs/git-repo-viewer-hub-script.md +++ b/docs/git-repo-viewer-hub-script.md @@ -1,7 +1,10 @@ # Git repo viewer — hub script -The hub script `clone_repo_and_upload_to_instance_storage` is published from -`windmill-integrations` and pinned in `frontend/src/lib/hubPaths.json` as +The hub script `clone_repo_and_upload_to_instance_storage` +([hub page](https://hub.windmill.dev/scripts/windmill/13968)) is published from +`windmill-integrations` +(`hub/windmill/scripts/action/13968_clone_repo_and_upload_to_instance_storage/script.ts`) +and pinned in `frontend/src/lib/hubPaths.json` as `cloneRepoToS3forGitRepoViewer`. Hub paths are exact version pins, so editing the script means publishing a new version and repointing that entry. @@ -23,6 +26,11 @@ The repo viewer in the Windmill app expects the hub script to: 3. **Write a completion marker** as the very last action of a successful run, so the API and frontend can distinguish a fully-populated S3 directory from a partial / interrupted upload. +4. **Follow symlinks that stay inside the checkout.** Both the git clone and + the archive extraction keep a repository's symlinks as links, and + `Dirent.isFile()` / `isDirectory()` are both false for a link, so a walk + that only checks those drops every linked file and directory from the + viewer. See [Symlinks](#symlinks). The marker file the frontend looks for is `.windmill_clone_complete` at the root of the per-commit directory: @@ -43,28 +51,74 @@ after the walk completes: ```ts const UPLOAD_CONCURRENCY = 16 const CLONE_MARKER_FILE = ".windmill_clone_complete" +const MAX_SYMLINKED_ENTRIES = 20_000 +const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024 async function uploadDirectoryToS3( directoryPath: string, s3BasePath: string, workspace: string, -) { +): Promise { console.log(`Uploading ${directoryPath} -> ${s3BasePath}`) // Walk the directory once, producing a flat list of (localPath, s3Key) pairs. const tasks: { localPath: string; s3Key: string }[] = [] - function walk(dir: string, s3Path: string) { + const root = fs.realpathSync(directoryPath) + // Real paths of the directories being descended through. + const ancestors = new Set() + // What entries reached through a link have cost so far; see Symlinks below. + let symlinkedEntries = 0 + let symlinkedBytes = 0 + let symlinkBudgetSpent = false + function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean { + if (symlinkBudgetSpent) return false + symlinkedEntries += entries + symlinkedBytes += bytes + if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) { + return true + } + symlinkBudgetSpent = true + console.log( + `Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` + + `${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB` + ) + return false + } + function walk(dir: string, relDir: string, viaLink: boolean) { + ancestors.add(dir) for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name) - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name - if (entry.isDirectory()) { - walk(fullPath, s3Key) - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }) + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name + const linked = viaLink || entry.isSymbolicLink() + if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue + let localPath = join(dir, entry.name) + if (entry.isSymbolicLink()) { + const link = fs.readlinkSync(localPath) + try { + localPath = fs.realpathSync(localPath) + } catch (e: any) { + console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`) + continue + } + if (localPath !== root && !localPath.startsWith(root + sep)) { + console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`) + continue + } + } + const stat = fs.statSync(localPath) + if (stat.isDirectory() && ancestors.has(localPath)) { + console.log(`Skipping ${relPath}: links back to a directory it is inside`) + continue + } + if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue + if (stat.isDirectory()) { + walk(localPath, relPath, linked) + } else if (stat.isFile()) { + tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` }) } } + ancestors.delete(dir) } - walk(directoryPath, s3BasePath) + walk(root, "", false) console.log(`Discovered ${tasks.length} files to upload`) @@ -114,9 +168,40 @@ async function uploadDirectoryToS3( requestBody: new Blob([markerBody], { type: "application/json" }), }) console.log(`Wrote completion marker: ${markerKey}`) + + return tasks.length } ``` +## Symlinks + +A link is resolved with `realpathSync` and followed only when its target lies +inside the checkout's real path. A file target is uploaded under the link's own +path; a directory target is walked as if it sat there, so +`inventories/prod/group_vars -> ../../shared/group_vars` shows up in the viewer +with its files. Everything else is skipped and logged: + +- **A target outside the checkout.** The repository chooses the target, and the + checkout sits in the job's working directory next to the ssh key + `get_git_ssh_cmd` writes (`../ssh_id_priv_0`) and the job's `args.json`. A + link to one of those, or to `/proc/self/environ` with the caller's + `WM_TOKEN`, would put it in storage for every reader of the resource. This + is why the walk does not follow links the way `aws s3 sync` does. +- **A target that cannot be resolved**: a dangling link, or a link loop + (`ELOOP`). +- **A directory that is already being walked higher up** (`loop -> .`, + `up -> ..`). The guard holds the real paths of the current descent only, as + `find -L` does, not every directory seen so far: a directory reachable + through two links is uploaded under both paths, as the checkout presents it. +- **Anything reached through a link once the budget is spent.** Because a + directory can be reached along many paths, two links to the next directory + at each level double the tree, and a repository a few dozen links deep would + expand past what the job can hold in memory. Every entry reached through a + link counts against a budget of 20,000 entries and 512 MiB. It is charged + before the link is resolved, so links that end up skipped count too, and + neither their work nor their log lines can multiply. Past the budget, the rest + are skipped with one log line. The checkout's own files are always uploaded. + ## Notes for review - **Concurrency level**: 16 is a starting point; tune based on instance @@ -127,6 +212,12 @@ async function uploadDirectoryToS3( paths on retry, so a partial upload + retry naturally heals. Old commit directories from before this patch are unreachable through the UI but still consume storage; an instance admin can prune them manually if desired. +- **A new pin doesn't refresh commits already uploaded**: the viewer keys + storage on the commit hash (`gitrepos/{workspace}/{resource_path}/{commit_hash}/`) + and only checks that the marker exists. So a commit uploaded by an earlier + script version keeps that version's tree (hub/28905's had no symlinks) until + the repository's head moves to a new commit, or an admin deletes that + commit's directory. - **Error propagation**: keep the existing `try/catch` in `main` so an upload failure surfaces in the job result and is shown in the new viewer error banner. diff --git a/docs/git-sync-gitlab-setup.md b/docs/git-sync-gitlab-setup.md new file mode 100644 index 0000000000..6e736dee37 --- /dev/null +++ b/docs/git-sync-gitlab-setup.md @@ -0,0 +1,170 @@ +# Git sync with GitLab + +GitLab has no equivalent of a GitHub App, so there is nothing to install and no +consent screen. What Windmill needs instead is one credential you create in +GitLab and paste once. With it, a GitLab repository gets the same managed +features an app-backed GitHub repository has: instant pull over a webhook, merge +requests opened on deploy, and a diff preview posted onto the merge request. + +## The credential + +Create a **project access token** on the project you are syncing (Settings → +Access tokens). It is a bot identity that outlives the person who created it, +which is what you want for a credential the instance uses unattended, and it +reaches exactly the one project. + +| | | +| --- | --- | +| Scope | `api` | +| Role | Developer to push deploy branches; **Maintainer** to also manage the webhook and open merge requests | +| Expiry | Required. A group service account PAT can be non-expiring on self-managed (see below); an access token cannot | + +**Use a separate token per repository.** A group access token works too and +reaches every project in the group, which is convenient for a lot of +repositories — but Windmill stores the credential per repository, and renewal +rewrites the repository it renewed for. Any other repository holding that same +token keeps the revoked one and stops syncing until you paste a new token there. +Each stranded repository says so on its card, so it is visible rather than +silent, but a token per repository avoids it entirely. + +`api` is a superset: it authorizes Git over HTTPS as well, so no separate +`write_repository` is needed to clone and push, and it is also what makes the +token renewable. A `write_repository`-only token can still push, but Windmill +cannot inspect or renew it and reports that in the workspace's git sync settings. + +### The identity Windmill acts as + +GitLab issues an access token to a bot user it creates for it — `project__bot_…` +for a project token, `group__bot_…` for a group one — and the bot's display +name is **the name you gave the token**. That name is the byline on everything +Windmill does: the author of deploy commits, of the merge requests it opens, and +of the preview notes it writes. Name it for what it is, `windmill-sync` or +similar, rather than something only you will recognise. + +Each token you create adds another bot member to the project or group. Renewal +does not — it keeps the same bot — so a repository accumulates one bot, not one +per year. + +Renewal goes through GitLab's own self-rotation endpoint. Both kinds of access +token are held as their bot user's personal access token, so the token rotates +itself and Windmill never needs a credential with rights over the project or +group. + +## Connecting a repository + +In the resource form for a `git_repository` resource, use the **GitLab** button: +paste the instance URL and the token, pick a project from the list, and Windmill +keeps the token for you. The resource itself gets the plain remote URL +(`"url": "https://gitlab.com/group/project.git"`), with no credential in it. + +The token is stored encrypted on the workspace, keyed by the repository it was +issued for rather than by the resource naming it, the same way a GitHub App +installation is held against the account it covers. Nothing reads it back out +over the API: the server attaches it when it talks to GitLab, and a sync job +receives it only against its own job token. Repointing a resource's `url` asks +for a different repository's token and finds none, so the edit carries nothing +with it; a repository that genuinely moved needs its token entered again. + +Because the repository is the key, the token is stored the moment you pick the +project, before the resource is saved. Renaming the resource later keeps it, and +cancelling the edit leaves a stored token that nothing uses until some resource +points at that repository again. + +Forks of the workspace read this one copy rather than getting their own, so +renewal reaches all of them at once and the token is not duplicated into every +descendant workspace. + +Treat workspace admin as equivalent to holding the token. An admin of the +workspace, or of any fork below it, can point a repository at a sync script they +wrote and have that job request the credential, exactly as they can for a GitHub +App installation token. Storing it this way keeps it out of the variables API and +out of every fork's own storage; it is not a boundary against the admins of those +workspaces. + +A URL with the token written into it, in the resource or in a secret variable +the resource points at (`"url": "$var:..."`), is a plain git remote: it syncs on +deploy and by polling, and nothing else. Windmill does not know the token is +there, so it registers no webhook, opens no merge request, and neither reports +nor renews its expiry. Use the **GitLab** button to hand the token to Windmill +if you want any of that. + +## Expiry and renewal + +Windmill reads `expires_at` from the token it holds and shows it on the +repository in the workspace's git sync settings. Within three weeks of expiry it +rotates the token through GitLab's own +`POST /personal_access_tokens/self/rotate`, stores the replacement, and verifies +it. Only the token can rotate itself, so one without `api` (or `self_rotate`) is +a permanent warning rather than something Windmill can fix. + +Only the workspace that holds a credential renews it, so one renewal serves the +whole fork chain instead of each fork racing to renew the same token. A fork +reads the parent's without holding one, so it never renews; the parent does, and +every fork sees the replacement at once. + +Rotation is deliberately never retried. GitLab revokes the old token the instant +it issues the replacement, and presenting an already-rotated token to `/rotate` +again is treated as reuse: it revokes **the whole token family, including the +live replacement**. So a rotation that succeeded at GitLab but failed to persist +is surfaced as an error to act on, not retried. + +Non-expiring tokens are possible only for a **group service account PAT** on +self-managed, with `require_personal_access_token_expiry` turned off in the +instance's application settings. A group access token is always rejected without +an `expires_at`. + +## What each managed feature needs + +| Feature | Needs | +| --- | --- | +| Instant pull | A project hook Windmill creates, so Maintainer; and a Windmill base URL GitLab can reach | +| Merge requests on deploy | Developer, plus the `api` scope | +| Diff preview on a merge request | The project hook, plus permission to post merge request notes | + +Instant pull falls back to checking the tracked branch about every minute when +the hook cannot be created or delivered, so nothing silently stops syncing. + +## Self-managed differences + +**Webhooks to a private network are blocked by default.** GitLab refuses to +create a hook pointing at a private or local address until an administrator +enables *Allow requests to the local network from webhooks and integrations* +(Admin → Settings → Network → Outbound requests, +`allow_local_requests_from_web_hooks_and_services`). A Windmill instance on the +same private network as GitLab needs this; without it, hook creation fails with a +"blocked" error and the repository keeps polling. + +**A relative-URL install is not supported.** GitLab can be served under a path +prefix (`https://example.com/gitlab`), and that prefix cannot be told apart from +a group of the same name: `example.com/a/b/c.git` is either group `a/b` project +`c`, or prefix `a` with group `b` project `c`. Windmill reads it as the nested +group, so on a relative-URL install it derives the wrong API base and the managed +features stay unavailable. Such a repository still syncs through its token URL, +which needs no API base. + +Everything else is identical: Windmill talks to `/api/v4` and needs +no inbound access of its own beyond the hook deliveries. + +## The deploy preview is a note, not a pipeline status + +On GitHub the preview is a check run: its own object, advisory unless the +repository makes it required. GitLab has no equivalent. Its only comparable +primitive is a commit status, and posting one has side effects Windmill will not +impose on a project: + +- GitLab files the status **as a job inside whatever pipeline already covers that + commit**, so a failed Windmill status fails the project's own pipeline, and its + reviewers see their test suite as failed. +- `allow_failure` is ignored on the commit-status endpoint, so the status cannot + be made advisory. +- On a commit with no pipeline it creates an `external` pipeline instead, which + then gates merging under *Pipelines must succeed*, including while it is still + running. + +So on GitLab the preview lives entirely in a **merge request note** that Windmill +keeps up to date: it carries the workspace, the status line, the commit, a link +to the job, and the full list of changes merging would deploy. A note cannot +block a merge or change what the project's own CI reports. + +The note is upserted rather than appended, so a merge request accumulates one +Windmill comment however many times it is pushed to. diff --git a/docs/git-sync-pull-design.md b/docs/git-sync-pull-design.md index 8961b0f047..53d36c82fe 100644 --- a/docs/git-sync-pull-design.md +++ b/docs/git-sync-pull-design.md @@ -186,6 +186,35 @@ Routing — an event/poll result is `(repo, ref, head_sha, sender)`: filters): fan out, each workspace pulls with its own filters; `wmill.yaml` in the repo stays authoritative for include/exclude. +Identity — a pull applies changes as a real workspace admin, never a reserved identity. +The schedules, triggers and app policies it deploys persist their deployer as the identity +they run as, and `validate_on_behalf_of` refuses reserved sentinels there, so a real admin +is what keeps those deployable and revocable (demote or remove the admin and what runs +under them stops). + +- The admin is `auto_pull.enabled_by`, stamped server-side with the email of whoever last + saved the git sync settings with auto pull on. Re-saving as another admin rotates it. +- A stamp naming someone who is no longer an active admin (demoted, or deactivated in the + workspace or on the instance), and not an active instance superadmin either, fails the + pull rather than falling back to someone else. A superadmin who is not a member runs it + under their instance username, and only while no member of the workspace holds that + username: `u/` resolves through the workspace's members before the email. A + repository whose settings predate the stamp runs as the workspace's first active admin + until they are saved again. +- The identity is resolved before the deploy check is posted, and a failure to resolve it + or to enqueue is recorded on the repository's status, not returned: a returned error + would fail the webhook delivery, and hosts disable hooks whose deliveries keep failing. + The next push or poll retries. +- Fork pulls run as the parent repository's identity, stamped or not, resolved in the + parent (revoking that admin there stops fork pulls too), and first add that admin to + the fork as an admin member, since a plain fork carries only its creator. The fork's + owner cannot be the identity: a non-admin's `wmill sync push` diffs against what it can + see, so an item in a folder it cannot read reads as a create and the push fails on every + commit. CI tests do run as the owner (Phase 7), because they only execute. +- Known and accepted: repo writers control the pull's includes through `wmill.yaml`, so a + fork's owner can commit a user file that makes them admin of the fork and read the + parent secrets it cloned, as with the `push-on-merge-to-forks` Action this replaces. + Loop prevention (pull → deploys → deployment callback → commit → push event): 1. Skip events whose sender is the app bot (`windmill-sync-helper[bot]` / @@ -618,6 +647,93 @@ repo's **Environments** timeline ("Production → Deployed"). Needs opt-in / later. The check-run version is the cheap default and matches the visual Cloudflare parity without a new permission. +### Phase 7 — CI test results check (WIN-2051) — implemented + +Surfaces Windmill's own CI tests (the `// test: script/...` annotation) as a +**"Windmill CI tests"** check run on **any PR** against the tracked branch, so a customer +can mark it a **required status check** and have Windmill CI results gate the PR — +replacing the documented GitHub Action that polls `ci_test_results_batch`. GitHub App-backed +only; reuses the Phase 4 `Checks: write` grant, so no new permission. Token repos keep the +Action, and GitLab merge requests get no CI-test surface for the same reason the Phase 4 +preview lives in a note there (a commit status would fail the project's own pipeline). + +Driven by the **`pull_request` webhook** — the same event Phase 4 already reacts to — +rather than the deploy push/pull, so it's uniform across how the PR's commit came to exist +(a fork deploy that pushes `wm-fork/**` and opens the PR, or an external push that gets +pulled in). CI tests run as separate async `ci_test` jobs in the **fork workspace** the PR +corresponds to; the check reflects that fork's current results on the PR head. + +- **State** — `git_sync_ci_test_check(workspace_id, repo_resource_path, head_sha)` (new + table). `workspace_id` is the **fork** whose `ci_test` jobs the check reflects; + `repo_resource_path` the repository (a fork can sync several, and two can hold the same + commit); `poster_workspace_id` is the **parent** whose GitHub-App installation posts the + run (the workspace that received the webhook and owns the repo hook). Plus `repo_url`, + `head_ref`, `check_run_id` (NULL until the create succeeds, and reset to NULL by a re-fired event + for the same head: the row is written first so a create that never gets recorded cannot + strand an in-progress run, and the poller retries any row without an id), `created_at`, + `concluded`, `conclusion`, `concluded_at`, `github_posted`. + Partial index `(workspace_id) WHERE NOT concluded OR NOT github_posted` (the live set the + hook + poller scan). +- **Open** — in the `pull_request` handler (opened/synchronize/reopened, or edited with a + base change, base = tracked): when the head lives in the base repo, resolve the fork + workspace from the head ref (reusing the fork-branch routing; + `resolve_pr_head_workspace`), persist the intent row with a null check-run id, then + `create_check_run` in_progress on `head_sha` via the parent's installation and adopt the + id (the poller retries the create from the row if it failed), then evaluate. An earlier head's open check is left to conclude on its + own (fork verdict or timeout): a late-delivered event for an old head must never touch + the current head's check. +- **Conclude** — the verdict is the head's own suite. Once the fork reflects the head (below) + and its dependency jobs settled, every CI test the fork declares is dispatched once, one + run per `ci_test_reference` row the way a deploy of that item would (`trigger_all_ci_tests`, + as the fork's owner, the user who created it, with no more reach than they have; a + missing or disabled owner concludes the check as failure; and without the per-item + debounce so a deploy-triggered run of the same test cannot supersede a suite run), and the job ids are + recorded on the synced-head row (`ci_test_job_ids`; `tests_dispatched_at` claims the + dispatch so the per-job hook and the poller queue it once, and a claim that never recorded + ids is retaken after 5 min). The verdict is exactly those runs: fail-fast on any + failed/canceled; `success` once all settle ("No CI tests" when the fork declares none); + `skipped` ignored. Nothing older, newer or workspace-wide stands in + for a head's runs, so a re-fired event reads the same runs and gets the same answer, a + test-only change is run because the suite runs on every head, and a deploy in flight in + the fork cannot feed another head's check. Runs purged by job retention reset the row so + the head is re-tested. +- **Readiness** — the suite is dispatched only once the fork reflects the head, so it does + not matter which webhook GitHub delivers first. The evidence is `git_sync_synced_head`: the + pull completion hook writes a row when a pull job succeeds (the pull script reports the + commit its clone checked out; the enqueue-time marker is the fallback), and the push + completion hook writes one from the deploy push script's `{pushed, sha, branch, rebased}` + result (a rebased push sits on unpulled commits and is not recorded). The head is ready + when the repository branch's newest row names it, so a branch reset to an older commit + waits for its re-pull; the prune keeps each repository branch's newest row so a PR reopened + at an unchanged head stays ready. This is a sync event log, deliberately apart from + `auto_pull.last_synced_sha`: that map decides whether the next poll pulls (a push must + never write it, or a commit someone else pushed under ours would be skipped) and it is + client-round-tripped settings. The check row stores `head_ref` for the lookup. Dispatch + also waits while a dependency job in the fork or a pull of the repository branch is queued + (a deploy push lands on whatever the remote held when it cloned, so a commit pushed there + from outside is in the workspace only once its pull ran), and the check fails outright if + a dependency job failed after the head's pull started (the item deployed nothing + runnable). A commit the fork never comes to reflect times out; a timeout on a repository + pinned to a sync script older than the one that reports pushed commits names that as the + reason. Needs the hub script versions that report the sha (`LATEST_GIT_SYNC_SCRIPT_PATH`, + `GIT_SYNC_PULL_SCRIPT_PATH`). +- **Drivers** — a per-`ci_test`-job completion hook (low latency) and the git-sync poller + (the backstop: retries the GitHub create/deliver, times stuck checks out after 30 min, + prunes old rows; runs after the auto-pull advisory lock is released so its GitHub calls + never extend the tick). Both call one idempotent `evaluate_and_conclude`, which claims the + decision with a guarded `UPDATE ... WHERE NOT concluded RETURNING` (exactly-once) and + decouples GitHub delivery via `github_posted` so a failed PATCH is retried, not hung. + +Invariants: only a head in the base repo can map to a workspace (a contributor fork's +branch names mean nothing here); the webhook's workspace posts through its own +installation; the timeout stops a hung test job from blocking a required check forever; +rows cascade away with either workspace. A plain feature-branch or +contributor-fork PR resolves to no fork workspace and gets an already-concluded `skipped` +check (branch protection counts `skipped` as passing, so requiring the check does not block +those PRs). A timeout on a repository pinned to a sync script older than the one that reports +pushed commits names that as the reason. Known limit (accepted for v1): the fork's status is workspace-wide (all its +tested items), which for the one-fork-per-PR model equals the PR's scope. + ## 16. Alternatives considered **Portal as webhook proxy (the rejected "option 2").** Subscribe the managed app diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8383bc2836..2f703bc0b5 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -16,28 +16,116 @@ every workspace via the standard cached-resource-type sync, like other built-in - The brain config and tools are resolved at runtime from the resource (`windmill-worker/src/ai_executor.rs`): the brain is interpolated, so a nested provider `$res:` credential resolves automatically. -- The step keeps only the flow-local inputs (`user_message`, `user_attachments`) in its own - `input_transforms`; the brain and tools stay in the resource (read-only in the step). +- The step keeps only the flow-local inputs (`user_message`, `user_attachments`, `enabled_tools`, + and the history inputs `memory_id` and `previous_messages`) in its own `input_transforms`; the + brain and tools stay in the resource (read-only in the step). `enabled_tools` says which of the + roster this step may call, narrowing one use of a shared agent without touching the agent: an + absent field carries every tool, a list carries the ones it names, and an empty list carries + none. - The agent carries its tools' default input bindings verbatim as authored (static, AI-filled, or flow expressions), so saving round-trips losslessly. Each host flow overrides what it needs: `tool_inputs` stores per-tool overrides (a diff from the resource tool's own transforms) that overlay onto the matching tools at runtime. Editing on a linked step edits - the flow's use of the agent; editing under the "Editing" banner edits the agent itself. + the flow's use of the agent; editing in the agent editor edits the agent itself. In the flow editor, the AI agent step's **Step Input** tab shows a single read-only card (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus -*Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* -(fork the resolved config — including any `tool_inputs` — back into the step as a one-off). -While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking -first when there is something to drop), and the unsaved-changes badge opens a diff against the -deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, -and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so -a reload brings the step back as a standalone agent with no path to save back to. +*Edit*, which opens the agent editor over the flow, and *Unlink* (fork the resolved config — +including any `tool_inputs` — back into the step as a one-off). A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into `tool_inputs`. +## Memory + +Memory is split between three owners, so a saved agent carries whether it remembers and never +which memory it is: + +- **Agent: managed memory.** `memory` is a brain key, so it moves with a saved agent. + `{ kind: window, context_length }` has Windmill store the conversation and replay its last N + messages; `{ kind: off }` keeps none. An absent `memory` means off, the default: the editor turns + it on when chat input is enabled. `auto` and `manual` are the older spellings and are still read. +- **Run: memory id.** `flow_status.memory_id`, set when the run is queued: the chat conversation + id, an app chat session id, or the `memory_id` run parameter. Any string is accepted, and one + that is not a uuid is hashed to a v5 uuid scoped to the workspace and the flow the run started + from (`memory_key` in `windmill-common/src/flow_conversations.rs`), so the same key in two flows + names two memories. A uuid is used as is. Nothing is generated at save time, so schedules, + webhooks, evals and plain runs pass no id and run stateless. +- **Step: history inputs.** Flow-local, so they stay on a linked step. Each is read in one memory + state only, and the editor offers it only there, the memory id behind a *Custom* toggle that + writes the key only once it is on. With managed memory on, `memory_id` overrides the run's id, + hashed the same way: a fixed value is one memory shared by every run, an expression such as + `flow_input.customer_id` one memory per key, and an expression that evaluates to nothing runs + stateless rather than falling back to the run's id. With memory off, `previous_messages` supplies + the history itself. An older `auto` or `manual` memory reads neither, so the editor offers them + only once the step is moved to the current settings, which the alert's button does. The editor + never seeds a placeholder for either, because a present key is the step's choice, and a static + empty value reads as unset. + +The worker reconciles them once per agent invocation, nested agent tools included, in +`resolve_history_source` (`windmill-worker/src/ai_executor.rs`): + +1. A legacy `auto` or `manual` memory: read as the editor that wrote it ran it. `manual` replays + its list; `auto` uses the run's memory id, else the id baked into it, else runs stateless. + Neither history input is read. An `auto` without a count, or with 0, is off and read as such. +2. Managed memory: the memory id is the step's, else the run's. With no memory id the agent runs + stateless, and a step `previous_messages` is ignored. +3. Memory off: the history is `previous_messages`, else nothing. Memory is neither read nor + written, and a step `memory_id` is ignored. + +Each ignored input and each stateless fallback is written to the job log. + +Memory is stored per (memory id, step id), in `ai_agent_memory` or S3 at +`memory/{workspace}/{memory id}/{step}.json`. The chat transcript (`flow_conversation_message`) +always follows the run's id, even when a step sets its own. Nothing expires stored memory: deleting +a chat conversation deletes its memory, and a memory named by a string id stays until it is +overwritten. + +Compatibility runs one way. New workers read every older shape. The editor rewrites a legacy step +only when the author changes it, so a flow nobody edits keeps running on older workers, while a +step saved with `window` or a history input needs a worker that knows them. An id an older editor +baked into `memory` stays a fallback behind the run's id until the author chooses *Keep as memory +id* or *Use the run's memory id*. In a chat flow it is dropped on save, since the conversation id +always took precedence there. + +## Drafts + +The agent editor edits the resource through a **per-user resource draft** (`draft` table, +`item_kind = 'resource'`), autosaved by `useAgentDraft` and deployed by the editor's own Deploy +button. It is the same draft row the generic resource editor writes and the Review & Deploy page +lists, so an agent can be deployed from any of them. + +A flow does not wait for that deploy to see the draft: + +- Testing the flow, or a single linked step, runs the draft. `runFlowPreview` and `ModuleTest` + substitute each linked step for the standalone step the draft would run as + (`linkedAgentDrafts.ts`): `agent` cleared, the draft's brain as static input transforms, the + draft's tools on the step, and the step's own flow-local inputs kept on top — + the same overlay order `ai_executor.rs` applies to a linked step. `tool_inputs` is untouched, + since the worker overlays it in both branches. +- The step's linked card and the graph's tool nodes show the draft, with a *Draft* badge, so the + editor describes what a test would run. Read-only surfaces (the deployed flow page, the run + viewer) stay on the deployed agent: they resolve tools through `publishLinkedAgentTools` without + the draft flag. +- Deploying the flow lists every linked agent that has a draft in the confirmation dialog, beside + the draft triggers. Deploying one writes the resource and drops the draft; leaving one out keeps + its draft untouched, and the flow runs the agent as currently deployed. That is the one place + the two kinds differ: an undeployed draft trigger is deleted, because it belongs to the flow, + while an agent draft belongs to a resource other flows also use. + +Because a draft is per-user, a flow test can behave differently for two people looking at the same +flow. That is the same contract as a flow draft, and deploying the agent is what makes it shared. + +Inlining has a consequence worth knowing: a preview job's `raw_flow` then carries the agent's +config, where a linked step used to carry only the path and leave the resolution to the worker. So +an agent's prompt and tool set are readable by whoever can read that preview job, which is a wider +set than whoever can read the resource when the agent sits in a more restricted folder than the +flow. No credential travels with it — the provider stays a `$res:` reference, resolved at run time +as the runner. The agent editor's own test pane has inlined the same way since drafts existed; +closing the gap would mean the preview carrying a draft *reference* the worker resolves, rather +than the config. + Sharing works through standard resource folder permissions (save agents under `f/...`). Only the agent's brain is interpolated when the step runs. A tool's own `$res:`/`$var:` defaults are diff --git a/docs/wac-sdk-e2e.md b/docs/wac-sdk-e2e.md index 5c6682d93d..0920bdf168 100644 --- a/docs/wac-sdk-e2e.md +++ b/docs/wac-sdk-e2e.md @@ -34,8 +34,12 @@ cp python-client/wmill/wmill/client.py \ /tmp/windmill-mytest/cache/python_3_12/wmill==*/wmill/client.py find /tmp/windmill-mytest -name __pycache__ -type d -exec rm -rf {} + -# 4. RESTART the backend — see below -# 5. run your scenarios, and rm -rf /tmp/windmill-mytest when done +# 4. drop the bundle snapshots — a bun job runs a bundle built from the package, +# cached by content hash, so a job that already ran keeps the old SDK inlined +rm -rf /tmp/windmill-mytest/cache/bun + +# 5. RESTART the backend — see below +# 6. run your scenarios, and rm -rf /tmp/windmill-mytest when done ``` ## Restart the workers after injecting diff --git a/flake.nix b/flake.nix index 9062fdb4fd..f005644e3d 100644 --- a/flake.nix +++ b/flake.nix @@ -65,7 +65,10 @@ # Misc libtool - postgresql + # Must not trail the server the dev database runs (postgres:18): pg_dump refuses a + # server newer than itself by a major version, which takes out every data table + # export, clone and fork-with-data. + postgresql_18 # Build tooling pkg-config diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 82f5133ddb..072729566c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.798.1", + "version": "1.816.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.798.1", + "version": "1.816.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -19,8 +19,10 @@ "@codingame/monaco-vscode-standalone-json-language-features": "=25.0.0", "@codingame/monaco-vscode-standalone-languages": "=25.0.0", "@codingame/monaco-vscode-standalone-typescript-language-features": "=25.0.0", + "@floating-ui/core": "^1.3.1", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", + "@melt-ui/svelte": "^0.86.2", "@popperjs/core": "^2.11.8", "@redocly/json-to-json-schema": "^0.0.1", "@scalar/openapi-parser": "^0.15.0", @@ -29,6 +31,8 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "@zerodevx/svelte-toast": "^0.9.6", + "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", @@ -38,10 +42,11 @@ "chartjs-adapter-date-fns": "^3.0.0", "chartjs-plugin-zoom": "^2.0.0", "clone": "^2.1.2", + "d3-dag": "^0.11.5", "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", - "diff": "^7.0.0", - "dompurify": "^3.3.1", + "diff": "^8.0.3", + "dompurify": "^3.4.13", "driver.js": "^1.3.0", "esm-env": "^1.0.0", "fast-equals": "^5.0.1", @@ -53,8 +58,8 @@ "lru-cache": "^11.1.0", "lucide-svelte": "^0.540.0", "mdast-util-find-and-replace": "^3.0.2", - "mermaid": "^11.15.0", - "minimatch": "^10.0.1", + "mermaid": "^11.16.1", + "minimatch": "^10.2.3", "modern-screenshot": "^4.7.0", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", @@ -66,15 +71,23 @@ "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", "quicktype-core": "^23.2.6", - "quill": "^1.3.7", + "quill": "^2.0.3", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "rfc4648": "^1.5.3", "runed": "^0.36.0", + "style-to-object": "^0.4.1", + "svelte": "^5.55.7", + "svelte-awesome-color-picker": "^3.0.4", "svelte-carousel": "^1.0.25", "svelte-exmarkdown": "^5.0.0", + "svelte-floating-ui": "^1.5.8", + "svelte-highlight": "^7.6.0", "svelte-infinite-loading": "^1.4.0", + "svelte-popperjs": "^1.3.2", + "svelte-range-slider-pips": "^2.3.1", + "svelte-splitpanes": "^8.0.9", "tailwind-merge": "^1.13.2", "unist-util-visit": "^5.0.0", "vscode": "npm:@codingame/monaco-vscode-extension-api@=25.0.0", @@ -101,33 +114,28 @@ "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", - "yaml": "^2.8.0", + "yaml": "^2.8.3", "yjs": "^13.6.7", "zod": "^4.1.10" }, "devDependencies": { - "@floating-ui/core": "^1.3.1", "@hey-api/openapi-ts": "^0.43.0", "@melt-ui/pp": "^0.3.2", - "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.57.0", "@sveltejs/adapter-static": "^3.0.6", - "@sveltejs/kit": "^2.53.4", + "@sveltejs/kit": "^2.69.1", "@sveltejs/package": "^2.5.7", "@sveltejs/vite-plugin-svelte": "^7.0.0", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", "@types/d3-zoom": "^3.0.3", - "@types/diff": "^7.0.1", "@types/lodash": "^4.14.195", "@types/vscode": "^1.83.5", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.60.0", - "@zerodevx/svelte-toast": "^0.9.6", "autoprefixer": "^10.4.13", "cssnano": "^6.0.1", - "d3-dag": "^0.11.5", "dts-bundle-generator": "^9.5.1", "eslint": "^8.47.0", "eslint-config-prettier": "^8.6.0", @@ -141,25 +149,17 @@ "postcss-load-config": "^4.0.1", "prettier": "^3.1.0", "prettier-plugin-svelte": "^3.3.3", - "style-to-object": "^0.4.1", "stylelint-config-recommended": "^13.0.0", - "svelte": "^5.53.5", - "svelte-awesome-color-picker": "^3.0.4", "svelte-check": "^4.4.3", "svelte-fast-check": "^0.4.5", - "svelte-floating-ui": "^1.5.8", - "svelte-highlight": "^7.6.0", - "svelte-popperjs": "^1.3.2", "svelte-preprocess": "^6.0.0", - "svelte-range-slider-pips": "^2.3.1", - "svelte-splitpanes": "^8.0.9", "tailwindcss": "^3.4.1", - "tar": "^7.5.4", + "tar": "^7.5.18", "tslib": "^2.6.1", "typescript": "^5.5.0", "vite": "^8.2.0", "vite-plugin-mkcert": "^2.0.0", - "vitest": "^4.1.0", + "vitest": "^4.1.11", "vitest-browser-svelte": "^2.0.1" }, "optionalDependencies": { @@ -1151,9 +1151,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", "dependencies": { @@ -1175,9 +1175,9 @@ "license": "MIT" }, "node_modules/@eslint/eslintrc/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -1219,7 +1219,6 @@ "version": "1.7.3", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.3.tgz", "integrity": "sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/utils": "^0.2.10" @@ -1229,7 +1228,6 @@ "version": "1.7.4", "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.4.tgz", "integrity": "sha512-OOchDgh4F2CchOX94cRVqhvy7b3AFb+/rQXyswmzmGakRfkMgoWVjfnLWkRirfLEfuD4ysVW16eXzwt3jHIzKA==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/core": "^1.7.3", @@ -1240,7 +1238,6 @@ "version": "0.2.10", "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.10.tgz", "integrity": "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==", - "dev": true, "license": "MIT" }, "node_modules/@glideapps/ts-necessities": { @@ -1289,9 +1286,9 @@ } }, "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -1344,33 +1341,11 @@ "version": "3.10.0", "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.10.0.tgz", "integrity": "sha512-oxDR/NTEJ1k+UFVQElaNIk65E/Z83HK1z1WI3lQyhTtnNg4R5oVXaPzK3jcpKG8UHKDVuDQHzn+wsxSz8RP3aw==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" } }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1597,7 +1572,6 @@ "version": "0.86.6", "resolved": "https://registry.npmjs.org/@melt-ui/svelte/-/svelte-0.86.6.tgz", "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/core": "^1.3.1", @@ -1612,12 +1586,12 @@ } }, "node_modules/@mermaid-js/parser": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.1.1.tgz", - "integrity": "sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.2.1.tgz", + "integrity": "sha512-n12NohV3mrUyUL2o93IgG/ifeW9FTyeJn3zDxkhwa8MJ9Fxg3HQMlA3RiGmD/3UnJvheztkjjQAjA2T4LmUcpw==", "license": "MIT", "dependencies": { - "@chevrotain/types": "~11.1.1" + "@chevrotain/types": "~11.1.2" } }, "node_modules/@noble/hashes": { @@ -2044,9 +2018,9 @@ } }, "node_modules/@sveltejs/acorn-typescript": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.6.tgz", - "integrity": "sha512-4awhxtMh4cx9blePWl10HRHj8Iivtqj+2QdDCSMDzxG+XKa9+VCNupQuCuvzEhYPzZSrX+0gC+0lHA/0fFKKQQ==", + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", "license": "MIT", "peerDependencies": { "acorn": "^8.9.0" @@ -2063,18 +2037,18 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.53.4", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.53.4.tgz", - "integrity": "sha512-iAIPEahFgDJJyvz8g0jP08KvqnM6JvdW8YfsygZ+pMeMvyM2zssWMltcsotETvjSZ82G3VlitgDtBIvpQSZrTA==", + "version": "2.70.3", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.70.3.tgz", + "integrity": "sha512-UDvEYuZqAMbfB/oXIoqKvbKcb7YczK5zYrzmsGV1zRJk03jntwp8dXiYoIJotxAndsKvcPFtx9H1GRSKFdSHgg==", "devOptional": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.0.0", - "@sveltejs/acorn-typescript": "^1.0.5", + "@sveltejs/acorn-typescript": "^1.0.9", "@types/cookie": "^0.6.0", - "acorn": "^8.14.1", + "acorn": "^8.16.0", "cookie": "^0.6.0", - "devalue": "^5.6.3", + "devalue": "^5.8.1", "esm-env": "^1.2.2", "kleur": "^4.1.5", "magic-string": "^0.30.5", @@ -2092,7 +2066,7 @@ "@opentelemetry/api": "^1.0.0", "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0 || ^6.0.0-next.0 || ^7.0.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "typescript": "^5.3.3", + "typescript": "^5.3.3 || ^6.0.0", "vite": "^5.0.3 || ^6.0.0 || ^7.0.0-beta.0 || ^8.0.0" }, "peerDependenciesMeta": { @@ -2181,7 +2155,6 @@ "version": "0.5.17", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", - "dev": true, "license": "Apache-2.0", "dependencies": { "tslib": "^2.8.0" @@ -2239,16 +2212,6 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, - "node_modules/@trysound/sax": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/@tutorlatin/svelte-tiny-virtual-list": { "version": "3.0.16", "resolved": "https://registry.npmjs.org/@tutorlatin/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-3.0.16.tgz", @@ -2551,13 +2514,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/diff": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@types/diff/-/diff-7.0.2.tgz", - "integrity": "sha512-JSWRMozjFKsGlEjiiKajUjIJVKuKdE3oVy2DNtK+fUo8q82nhFZ2CPQwicAIkXrofahDXrWJ7mjelvZphMS98Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2641,7 +2597,8 @@ "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/@types/unist": { "version": "3.0.3", @@ -2986,31 +2943,31 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.0.tgz", - "integrity": "sha512-EIxG7k4wlWweuCLG9Y5InKFwpMEOyrMb6ZJ1ihYu02LVj/bzUwn2VMU+13PinsjRW75XnITeFrQBMH5+dLvCDA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.0", - "@vitest/utils": "4.1.0", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", - "tinyrainbow": "^3.0.3" + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/mocker": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.0.tgz", - "integrity": "sha512-evxREh+Hork43+Y4IOhTo+h5lGmVRyjqI739Rz4RlUPqwrkFFDF6EMvOOYjTx4E8Tl6gyCLRL8Mu7Ry12a13Tw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.0", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3019,7 +2976,7 @@ }, "peerDependencies": { "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "msw": { @@ -3031,26 +2988,26 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.0.tgz", - "integrity": "sha512-3RZLZlh88Ib0J7NQTRATfc/3ZPOnSUn2uDBUoGNn5T36+bALixmzphN26OUD3LRXWkJu4H0s5vvUeqBiw+kS0A==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { - "tinyrainbow": "^3.0.3" + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/runner": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.0.tgz", - "integrity": "sha512-Duvx2OzQ7d6OjchL+trw+aSrb9idh7pnNfxrklo14p3zmNL4qPCDeIJAK+eBKYjkIwG96Bc6vYuxhqDXQOWpoQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.0", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -3065,14 +3022,14 @@ "license": "MIT" }, "node_modules/@vitest/snapshot": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.0.tgz", - "integrity": "sha512-0Vy9euT1kgsnj1CHttwi9i9o+4rRLEaPRSOJ5gyv579GJkNpgJK+B4HSv/rAWixx2wdAFci1X4CEPjiu2bXIMg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.0", - "@vitest/utils": "4.1.0", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3088,9 +3045,9 @@ "license": "MIT" }, "node_modules/@vitest/spy": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.0.tgz", - "integrity": "sha512-pz77k+PgNpyMDv2FV6qmk5ZVau6c3R8HC8v342T2xlFxQKTrSeYw9waIJG8KgV9fFwAtTu4ceRzMivPTH6wSxw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -3098,15 +3055,15 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.0.tgz", - "integrity": "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.0", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.0.3" + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" @@ -3183,7 +3140,6 @@ "version": "0.9.6", "resolved": "https://registry.npmjs.org/@zerodevx/svelte-toast/-/svelte-toast-0.9.6.tgz", "integrity": "sha512-nHlTrCjverlPK9yukK6fqbG3e/R+f10ldrc4nJHOe2qNDScuPTuYVSFEk2dDDtzWAwTN5pmdEXgA3M2RbT8jiw==", - "dev": true, "license": "ISC", "peerDependencies": { "svelte": "^3.57.0 || ^4.0.0 || ^5.0.0" @@ -3220,9 +3176,9 @@ } }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -3273,9 +3229,9 @@ } }, "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -3555,13 +3511,16 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.8.16", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.16.tgz", - "integrity": "sha512-OMu3BGQ4E7P1ErFsIPpbJh0qvDudM/UuJeHgkAvfWe+0HFJCXh+t/l8L6fVLR55RI/UbKrVLnAXZSVwd9ysWYw==", + "version": "2.11.24", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.24.tgz", + "integrity": "sha512-hYrgxie335U08WqICoGqKRzV1HFXv6zdxwJE4ekCb80CM9a0SVVsN4QPwT67RraRo+9h8IATk6uxHJw7QSkdOg==", "dev": true, "license": "Apache-2.0", "bin": { - "baseline-browser-mapping": "dist/cli.js" + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" } }, "node_modules/bezier-easing": { @@ -3613,9 +3572,9 @@ "license": "ISC" }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -3650,9 +3609,9 @@ "license": "MIT" }, "node_modules/browserslist": { - "version": "4.26.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.26.3.tgz", - "integrity": "sha512-lAUU+02RFBuCKQPj/P6NgjlbCnLBMp4UtgTx7vNHd3XSIJF87s9a5rA3aH2yw3GS9DqZAUbOtZdCCiZeVRqt0w==", + "version": "4.29.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", + "integrity": "sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==", "dev": true, "funding": [ { @@ -3670,11 +3629,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.8.9", - "caniuse-lite": "^1.0.30001746", - "electron-to-chromium": "^1.5.227", - "node-releases": "^2.0.21", - "update-browserslist-db": "^1.1.3" + "baseline-browser-mapping": "^2.11.23", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.427", + "node-releases": "^2.0.55", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -3780,28 +3739,11 @@ "node": ">=8.10.0" } }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -3815,6 +3757,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -3936,9 +3879,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001750", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001750.tgz", - "integrity": "sha512-cuom0g5sdX6rw00qOoLNSFCJ9/mYIsuSOA+yzpDw8eopiFqcVwQvZHqov0vmEighRxX++cfC0Vg1G+1Iy/mSpQ==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -4184,7 +4127,6 @@ "version": "2.9.3", "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", - "dev": true, "license": "MIT" }, "node_modules/combined-stream": { @@ -4262,9 +4204,9 @@ "license": "MIT" }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "devOptional": true, "license": "MIT", "engines": { @@ -4710,7 +4652,6 @@ "version": "0.11.5", "resolved": "https://registry.npmjs.org/d3-dag/-/d3-dag-0.11.5.tgz", "integrity": "sha512-sNHvYqjzDlvV2fyEkoOCSuLs2GeWliIg7pJcAiKXgtUSxl0kIX0C2q1J8JzzA9CQWptKxYtzxFCXiKptTW8qsQ==", - "dev": true, "license": "MIT", "dependencies": { "d3-array": "^3.1.6", @@ -5256,26 +5197,6 @@ "dev": true, "license": "MIT" }, - "node_modules/deep-equal": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.2.tgz", - "integrity": "sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg==", - "license": "MIT", - "dependencies": { - "is-arguments": "^1.1.1", - "is-date-object": "^1.0.5", - "is-regex": "^1.1.4", - "object-is": "^1.1.5", - "object-keys": "^1.1.1", - "regexp.prototype.flags": "^1.5.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -5318,44 +5239,10 @@ "node": ">=6" } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/defu": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz", - "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==", + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", + "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", "dev": true, "license": "MIT" }, @@ -5405,9 +5292,9 @@ } }, "node_modules/devalue": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.6.3.tgz", - "integrity": "sha512-nc7XjUU/2Lb+SvEFVGcWLiKkzfw8+qHI7zn8WYXKkLMgfGSHbgCEaR6bJpev8Cm6Rmrb19Gfd/tZvGqx9is3wg==", + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.2.tgz", + "integrity": "sha512-po4PAY5c53tw5XMocSnf8A/5OHhbbUftpr93aEN6BBoAdntUmK7vu7wOATqvt7cXO7m1Cl4gMVn6p7n6n4mj0w==", "license": "MIT" }, "node_modules/devlop": { @@ -5442,9 +5329,9 @@ "license": "Apache-2.0" }, "node_modules/diff": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", - "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -5541,9 +5428,9 @@ } }, "node_modules/dompurify": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.1.tgz", - "integrity": "sha512-qkdCKzLNtrgPFP1Vo+98FRzJnBRGe4ffyCea9IwHB1fyxPOeNTHpLKYGd4Uk9xvNoH0ZoOjwZxNptyMwqrId1Q==", + "version": "3.4.15", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz", + "integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -5578,9 +5465,9 @@ } }, "node_modules/driver.js": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.3.6.tgz", - "integrity": "sha512-g2nNuu+tWmPpuoyk3ffpT9vKhjPz4NrJzq6mkRDZIwXCrFhrKdDJ9TX5tJOBpvCTBrBYjgRQ17XlcQB15q4gMg==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.8.0.tgz", + "integrity": "sha512-+8/IO7h1v14IzWh2GP60N7T3PFZweXwdn5e5POuxRSBoCYUojsBxzqawPeXh3YZIibRy7EehYNEyxe7slwwtdg==", "license": "MIT" }, "node_modules/dts-bundle-generator": { @@ -5604,6 +5491,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -5639,9 +5527,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.235", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.235.tgz", - "integrity": "sha512-i/7ntLFwOdoHY7sgjlTIDo4Sl8EdoTjWIaKinYOVfC6bOp71bmwenyZthWHcasxgHDNWbWxvG9M3Ia116zIaYQ==", + "version": "1.5.430", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.430.tgz", + "integrity": "sha512-e1QEj72Y4zd8RlNZVmoTg+iCOSVwpk05IOiiQwdrkwCSVlZfPthevErhE+nckGd2YbsXfp1SkisznhGVIXP2NQ==", "dev": true, "license": "ISC" }, @@ -5718,6 +5606,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5727,6 +5616,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5743,6 +5633,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -5962,9 +5853,9 @@ } }, "node_modules/eslint-plugin-svelte/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -6013,9 +5904,9 @@ } }, "node_modules/eslint/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", "dependencies": { @@ -6064,9 +5955,9 @@ "license": "MIT" }, "node_modules/eslint/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -6086,7 +5977,6 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/esm-env-robust/-/esm-env-robust-0.0.3.tgz", "integrity": "sha512-90Gnuw2DALOqlL1581VxP3GHPUNHX9U+fQ+8FNcTTFClhY5gEggAAnJ3q1b2Oq23knRsjv8YpNeMRPaMLUymOA==", - "dev": true, "license": "MIT", "dependencies": { "esm-env": "^1.0.0" @@ -6147,15 +6037,6 @@ "node": ">=4.0" } }, - "node_modules/esrap": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.3.tgz", - "integrity": "sha512-8fOS+GIGCQZl/ZIlhl59htOlms6U8NvX6ZYgYHpRU/b6tVSh3uHkOHZikl3D4cMbYM0JlpBe+p/BkZEi8J9XIQ==", - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - } - }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", @@ -6219,9 +6100,9 @@ } }, "node_modules/eventemitter3": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-2.0.3.tgz", - "integrity": "sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, "node_modules/events": { @@ -6276,9 +6157,9 @@ "license": "MIT" }, "node_modules/fast-diff": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.1.2.tgz", - "integrity": "sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", "license": "Apache-2.0" }, "node_modules/fast-equals": { @@ -6342,9 +6223,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", "funding": [ { "type": "github", @@ -6357,6 +6238,15 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fastdom": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fastdom/-/fastdom-1.0.12.tgz", + "integrity": "sha512-LB+xjSTEbjHE1cWsxu+tN2Xqr1kpi+V9aADI7sVM5ZMaXyYGPHULQMzpJMYqOTULK/73pUkWVzzObFRBkPr+hg==", + "license": "MIT", + "dependencies": { + "strictdom": "^1.0.1" + } + }, "node_modules/fastest-levenshtein": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", @@ -6372,7 +6262,6 @@ "version": "0.7.2", "resolved": "https://registry.npmjs.org/fastpriorityqueue/-/fastpriorityqueue-0.7.2.tgz", "integrity": "sha512-5DtIKh6vtOmEGkYdEPNNb+mxeYCnBiKbK3s4gq52l6cX8I5QaTDWWw0Wx/iYo80fVOblSycHu1/iJeqeNxG8Jw==", - "dev": true, "license": "Apache-2.0" }, "node_modules/fastq": { @@ -6444,9 +6333,9 @@ } }, "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", "dev": true, "license": "ISC" }, @@ -6454,7 +6343,6 @@ "version": "7.6.5", "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.5.tgz", "integrity": "sha512-7Ke1jyybbbPZyZXFxEftUtxFGLMpE2n6A+z//m4CRDlj0hW+o3iYSmh8nFlYMurOiJVDmJRilUQtJr08KfIxlg==", - "dev": true, "license": "MIT", "dependencies": { "tabbable": "^6.2.0" @@ -6478,17 +6366,17 @@ } }, "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" @@ -6556,15 +6444,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/functions-have-names": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -6603,6 +6483,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -6627,6 +6508,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -6705,9 +6587,9 @@ } }, "node_modules/glob/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -6810,6 +6692,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6860,9 +6743,9 @@ } }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -6902,22 +6785,11 @@ "node": ">=8" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6930,6 +6802,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -6948,9 +6821,10 @@ "license": "MIT" }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -7337,7 +7211,6 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==", - "dev": true, "license": "MIT" }, "node_modules/internmap": { @@ -7349,22 +7222,6 @@ "node": ">=12" } }, - "node_modules/is-arguments": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", - "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -7402,22 +7259,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-date-object": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", - "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -7508,24 +7349,6 @@ "@types/estree": "^1.0.6" } }, - "node_modules/is-regex": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", - "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "gopd": "^1.2.0", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -7575,7 +7398,6 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/javascript-lp-solver/-/javascript-lp-solver-0.4.24.tgz", "integrity": "sha512-5edoDKnMrt/u3M6GnZKDDIPxOyFOg+WrwDv8mjNiMC2DePhy2H9/FFQgf4ggywaXT1utvkxusJcjQUER72cZmA==", - "dev": true, "license": "Unlicense" }, "node_modules/jiti": { @@ -7603,10 +7425,20 @@ "peer": true }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -7751,9 +7583,9 @@ } }, "node_modules/jsdom/node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", "dev": true, "license": "MIT", "engines": { @@ -7844,9 +7676,9 @@ } }, "node_modules/json-refs/node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", "dev": true, "license": "MIT", "dependencies": { @@ -8545,9 +8377,9 @@ } }, "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/lodash-es": { @@ -8710,6 +8542,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -8895,9 +8728,9 @@ } }, "node_modules/mdast-util-to-hast": { - "version": "13.2.0", - "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", - "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", "license": "MIT", "dependencies": { "@types/hast": "^3.0.0", @@ -9009,26 +8842,27 @@ } }, "node_modules/mermaid": { - "version": "11.15.0", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.15.0.tgz", - "integrity": "sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw==", + "version": "11.17.2", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.17.2.tgz", + "integrity": "sha512-V6K3C8EBdEsPFZXSKMJe6ppQOENxuHARr9GvHX4hh47lAbhMRD9qf4oEK7LoaRQxULMa80/qt5gHO73aCleBBg==", "license": "MIT", "dependencies": { - "@braintree/sanitize-url": "^7.1.1", + "@braintree/sanitize-url": "^7.1.2", "@iconify/utils": "^3.0.2", - "@mermaid-js/parser": "^1.1.1", + "@mermaid-js/parser": "^1.2.1", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", - "cytoscape": "^3.33.1", + "cytoscape": "^3.34.0", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", - "dayjs": "^1.11.19", - "dompurify": "^3.3.1", + "dayjs": "^1.11.21", + "dompurify": "^3.3.3", "es-toolkit": "^1.45.1", - "katex": "^0.16.25", + "fastdom": "1.0.12", + "katex": "^0.16.47", "khroma": "^2.1.0", "marked": "^16.3.0", "roughjs": "^4.6.6", @@ -9696,20 +9530,41 @@ } }, "node_modules/minimatch": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", - "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", - "license": "ISC", + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "license": "BlueOak-1.0.0", "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" + "brace-expansion": "^5.0.8" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimatch/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/minimatch/node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -9895,10 +9750,9 @@ } }, "node_modules/nanoid": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", - "integrity": "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==", - "dev": true, + "version": "5.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", + "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==", "funding": [ { "type": "github", @@ -10021,11 +9875,14 @@ } }, "node_modules/node-releases": { - "version": "2.0.23", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.23.tgz", - "integrity": "sha512-cCmFDMSm26S6tQSDpBCg/NR8NENrVPhAJSf+XbxBG4rPFaaonlEoE9wHQmun+cls499TQGSb7ZyPBRlzgKfpeg==", + "version": "2.0.55", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.55.tgz", + "integrity": "sha512-mIrE/Cw9y+9Au6dS5vDKDhQza9YvG6w+ZrS6X+ZzA7yFW/soAeaups4Qzn1bL6g5FVy8WtP79+0j82oPIbqRjQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/normalize-package-data": { "version": "3.0.3", @@ -10138,31 +9995,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-is": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.6.tgz", - "integrity": "sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.7", - "define-properties": "^1.2.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -10355,9 +10187,9 @@ } }, "node_modules/parchment": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/parchment/-/parchment-1.1.4.tgz", - "integrity": "sha512-J5FBQt/pM2inLzg4hEWmzQx/8h8D0CiDxaG3vyp9rKrQRSDgBlhjdP5jQGgosEajXPSQouXGHOmVdgo7QmJuOg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parchment/-/parchment-3.0.0.tgz", + "integrity": "sha512-HUrJFQ/StvgmXRcQ1ftY6VEZUq3jA2t9ncFN4F84J/vN0/FPpQF+8FKXb3l6fLces6q0uOHj6NJn+2xvZnxO6A==", "license": "BSD-3-Clause" }, "node_modules/parent-module": { @@ -10734,9 +10566,9 @@ } }, "node_modules/postcss-calc/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -10952,9 +10784,9 @@ } }, "node_modules/postcss-merge-rules/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -11034,9 +10866,9 @@ } }, "node_modules/postcss-minify-selectors/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -11074,9 +10906,9 @@ } }, "node_modules/postcss-nested/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -11379,9 +11211,9 @@ } }, "node_modules/postcss-unique-selectors/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -11400,9 +11232,9 @@ "license": "MIT" }, "node_modules/postcss/node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", "devOptional": true, "funding": [ { @@ -11508,9 +11340,9 @@ } }, "node_modules/protocol-buffers-schema": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.0.tgz", - "integrity": "sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw==", + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.1.tgz", + "integrity": "sha512-VG2K63Igkiv9p76tk1lilczEK1cT+kCjKtkdhw1dQZV3k3IXJbd3o6Ho8b9zJZaHSnT2hKe4I+ObmX9w6m5SmQ==", "license": "MIT" }, "node_modules/prr": { @@ -11542,13 +11374,14 @@ } }, "node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { - "side-channel": "^1.1.0" + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" }, "engines": { "node": ">=0.6" @@ -11561,7 +11394,6 @@ "version": "1.6.1", "resolved": "https://registry.npmjs.org/quadprog/-/quadprog-1.6.1.tgz", "integrity": "sha512-fN5Jkcjlln/b3pJkseDKREf89JkKIyu6cKIVXisgL6ocKPQ0yTp9n6NZUAq3otEPPw78WZMG9K0o9WsfKyMWJw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.x" @@ -11675,31 +11507,32 @@ } }, "node_modules/quill": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/quill/-/quill-1.3.7.tgz", - "integrity": "sha512-hG/DVzh/TiknWtE6QmWAF/pxoZKYxfe3J/d/+ShUWkDvvkZQVTPeVmUJVu1uE6DDooC4fWTiCLh84ul89oNz5g==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/quill/-/quill-2.0.3.tgz", + "integrity": "sha512-xEYQBqfYx/sfb33VJiKnSJp8ehloavImQ2A6564GAbqG55PGw1dAWUn1MUbQB62t0azawUS2CZZhWCjO8gRvTw==", "license": "BSD-3-Clause", "dependencies": { - "clone": "^2.1.1", - "deep-equal": "^1.0.1", - "eventemitter3": "^2.0.3", - "extend": "^3.0.2", - "parchment": "^1.1.4", - "quill-delta": "^3.6.2" + "eventemitter3": "^5.0.1", + "lodash-es": "^4.17.21", + "parchment": "^3.0.0", + "quill-delta": "^5.1.0" + }, + "engines": { + "npm": ">=8.2.3" } }, "node_modules/quill-delta": { - "version": "3.6.3", - "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-3.6.3.tgz", - "integrity": "sha512-wdIGBlcX13tCHOXGMVnnTVFtGRLoP0imqxM696fIPwIf5ODIYUHIvHbZcyvGlZFiFhK5XzDC2lpjbxRhnM05Tg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-5.1.0.tgz", + "integrity": "sha512-X74oCeRI4/p0ucjb5Ma8adTXd9Scumz367kkMK5V/IatcX6A0vlgLgKbzXWy5nZmCGeNJm2oQX0d2Eqj+ZIlCA==", "license": "MIT", "dependencies": { - "deep-equal": "^1.0.1", - "extend": "^3.0.2", - "fast-diff": "1.1.2" + "fast-diff": "^1.3.0", + "lodash.clonedeep": "^4.5.0", + "lodash.isequal": "^4.5.0" }, "engines": { - "node": ">=0.10" + "node": ">= 12.0.0" } }, "node_modules/rbush": { @@ -11872,26 +11705,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/regexp.prototype.flags": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", - "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "define-properties": "^1.2.1", - "es-errors": "^1.3.0", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "set-function-name": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/rehype-github-alerts": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/rehype-github-alerts/-/rehype-github-alerts-3.0.0.tgz", @@ -12239,6 +12052,16 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/sax": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, "node_modules/saxes": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", @@ -12278,38 +12101,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/set-function-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", - "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "functions-have-names": "^1.2.3", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", @@ -12340,15 +12131,15 @@ } }, "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" }, @@ -12360,14 +12151,14 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -12645,6 +12436,12 @@ "dev": true, "license": "MIT" }, + "node_modules/strictdom": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strictdom/-/strictdom-1.0.1.tgz", + "integrity": "sha512-cEmp9QeXXRmjj/rVp9oyiqcvyocWab/HaoN4+bwFeZ7QzykJD6L3yD4v12K1x0tHpqRqVpJevN3gW7kyM39Bqg==", + "license": "MIT" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -12749,7 +12546,6 @@ "version": "0.4.4", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.4.4.tgz", "integrity": "sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==", - "dev": true, "license": "MIT", "dependencies": { "inline-style-parser": "0.1.1" @@ -12773,9 +12569,9 @@ } }, "node_modules/stylehacks/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -12906,9 +12702,9 @@ "peer": true }, "node_modules/stylelint/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "peer": true, @@ -12968,9 +12764,9 @@ "license": "MIT" }, "node_modules/sucrase/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", "dev": true, "license": "MIT", "dependencies": { @@ -12988,9 +12784,10 @@ } }, "node_modules/sucrase/node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", "dev": true, "license": "ISC", "dependencies": { @@ -13009,13 +12806,13 @@ } }, "node_modules/sucrase/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^2.0.1" + "brace-expansion": "^2.0.2" }, "engines": { "node": ">=16 || 14 >=14.17" @@ -13093,23 +12890,22 @@ } }, "node_modules/svelte": { - "version": "5.53.5", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.5.tgz", - "integrity": "sha512-YkqERnF05g8KLdDZwZrF8/i1eSbj6Eoat8Jjr2IfruZz9StLuBqo8sfCSzjosNKd+ZrQ8DkKZDjpO5y3ht1Pow==", + "version": "5.57.0", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.0.tgz", + "integrity": "sha512-NdbDn7fl4be1ViUG0oq/lvG6OZy3oENolV2ONjiqqsfVoeAfzaQAKUcEX3MrQod/Bebv1PgwET9rfXhgn9s4Kg==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", - "@sveltejs/acorn-typescript": "^1.0.5", + "@sveltejs/acorn-typescript": "^1.0.10", "@types/estree": "^1.0.5", - "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", - "devalue": "^5.6.3", + "devalue": "^5.8.1", "esm-env": "^1.2.1", - "esrap": "^2.2.2", + "esrap": "^2.2.12", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", @@ -13123,7 +12919,6 @@ "version": "3.1.4", "resolved": "https://registry.npmjs.org/svelte-awesome-color-picker/-/svelte-awesome-color-picker-3.1.4.tgz", "integrity": "sha512-tiFakxvSpCwodOSFW6CflnLVSldSlDY77rDipHfW7hvB+4gTP3y6uO+Dm9O9e7P5N7K+CNyNjjdjbZOHYkMyNQ==", - "dev": true, "license": "MIT", "dependencies": { "colord": "^2.9.3", @@ -13137,7 +12932,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/svelte-awesome-slider/-/svelte-awesome-slider-1.1.2.tgz", "integrity": "sha512-HFIWwq6rtX6aXyc6ns2R0P6pDLYIr6nRjTPyAddAiTAgZqsohAoEfOEwKKqMmZ6Q+CH48zzcFdyo6lFkBJvyQw==", - "dev": true, "license": "MIT", "peerDependencies": { "svelte": "^3.54.0 || ^4.0.0 || ^5.0.0" @@ -13195,9 +12989,9 @@ } }, "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "optional": true, @@ -13310,7 +13104,6 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/svelte-floating-ui/-/svelte-floating-ui-1.6.2.tgz", "integrity": "sha512-EC+DZtBey50P6l3NSzNQWon3cip8a1bzwdpmCdc45kymqEWL4BKhPemAq7SQ9QLebDPaMECW6YodxFbs2d+O/w==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/dom": "^1.6.13" @@ -13320,7 +13113,6 @@ "version": "7.8.4", "resolved": "https://registry.npmjs.org/svelte-highlight/-/svelte-highlight-7.8.4.tgz", "integrity": "sha512-aVp+Q0hH9kI7PlSDrklmFTF4Uj7wYj7UGuqkREnkXlqpEffxr2g6esZcMMaTgECdg5rb2mJqM88+nwS10ecoTg==", - "dev": true, "license": "MIT", "dependencies": { "highlight.js": "11.11.1" @@ -13336,7 +13128,6 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/svelte-popperjs/-/svelte-popperjs-1.3.2.tgz", "integrity": "sha512-fwrErlkvngL876WXRnL3OLlfk/n9YkZwwLxuKRpZOYCJLt1zrwhoKTXS+/sRgDveD/zd6GQ35hV89EOip+NBGA==", - "dev": true, "license": "MIT", "peerDependencies": { "@popperjs/core": ">=2", @@ -13403,14 +13194,12 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/svelte-range-slider-pips/-/svelte-range-slider-pips-2.3.1.tgz", "integrity": "sha512-P29PNqHld+SiaDuHzf98rLvhSYWXb3TVL9p7U5RG9UX8emUgypZgp9zuIIwpmIXysGQC6JG8duMc5FuaPnSVdg==", - "dev": true, "license": "MPL-2.0" }, "node_modules/svelte-splitpanes": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/svelte-splitpanes/-/svelte-splitpanes-8.0.9.tgz", "integrity": "sha512-L3oLXTC99M191FInTXJ/f/2i0welRql1QuVbPaU8iy6nvCR6X9VyjHCsCpLqKGWHwqkWo/AM9CQ1c0nzlb+MkA==", - "dev": true, "license": "MIT", "dependencies": { "esm-env-robust": "0.0.3" @@ -13419,6 +13208,38 @@ "svelte": "^4.2.19 || ^5.1.0" } }, + "node_modules/svelte/node_modules/@typescript-eslint/types": { + "version": "8.70.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.70.0.tgz", + "integrity": "sha512-asTOIYhDg4zdzOScCyaytrsV3cR6B4ecPQlXw/dJIm7J/MZTtCtfVII9JD8Geh4jTCrK/Xe6cg5UevoleMcoJQ==", + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/svelte/node_modules/esrap": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.3.7.tgz", + "integrity": "sha512-n2nf7fZR3c9yXf0BPEuHuXqT+KW0SJVj4cN5FMEkpCZ3scLjOQWpiccyCxVzCC2q1wubTghuEGzngJY/7Ah0Ow==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, "node_modules/svelte2tsx": { "version": "0.7.49", "resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.7.49.tgz", @@ -13442,19 +13263,19 @@ "peer": true }, "node_modules/svgo": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.2.tgz", - "integrity": "sha512-OoohrmuUlBs8B8o6MB2Aevn+pRIH9zDALSR+6hhqVfa6fRwG/Qw9VUMSMW9VNg2CFc/MTIfabtdOVl9ODIJjpw==", + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.5.tgz", + "integrity": "sha512-8SQMzdrvWaD8deUmrnYB+ASyxBVgWUOilg+A75nE/76WdLpj6LopCwiAVvkzkcqy/9b7t2Mg7faFLjg0ZRcZ3w==", "dev": true, "license": "MIT", "dependencies": { - "@trysound/sax": "0.2.0", "commander": "^7.2.0", "css-select": "^5.1.0", "css-tree": "^2.3.1", "css-what": "^6.1.0", "csso": "^5.0.5", - "picocolors": "^1.0.0" + "picocolors": "^1.0.0", + "sax": "^1.5.0" }, "bin": { "svgo": "bin/svgo" @@ -13488,7 +13309,6 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", - "dev": true, "license": "MIT" }, "node_modules/table": { @@ -13596,9 +13416,9 @@ } }, "node_modules/tailwindcss/node_modules/postcss-selector-parser": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", - "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", "dev": true, "license": "MIT", "dependencies": { @@ -13623,9 +13443,9 @@ } }, "node_modules/tar": { - "version": "7.5.11", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz", - "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==", + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { @@ -13785,9 +13605,9 @@ } }, "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -14020,9 +13840,9 @@ } }, "node_modules/undici": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.1.0.tgz", - "integrity": "sha512-E9MkTS4xXLnRPYqxH2e6Hr2/49e7WFDKczKcCaFH4VaZs2iNvHMqeIkyUAD9vM8kujy9TjVrRlQ5KkdEJxB2pw==", + "version": "8.10.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", + "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", "dev": true, "license": "MIT", "engines": { @@ -14155,9 +13975,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", - "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -14417,19 +14237,19 @@ } }, "node_modules/vitest": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.0.tgz", - "integrity": "sha512-YbDrMF9jM2Lqc++2530UourxZHmkKLxrs4+mYhEwqWS97WJ7wOYEkcr+QfRgJ3PW9wz3odRijLZjHEaRLTNbqw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.0", - "@vitest/mocker": "4.1.0", - "@vitest/pretty-format": "4.1.0", - "@vitest/runner": "4.1.0", - "@vitest/snapshot": "4.1.0", - "@vitest/spy": "4.1.0", - "@vitest/utils": "4.1.0", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -14440,8 +14260,8 @@ "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.0.3", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "bin": { @@ -14457,13 +14277,15 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.0", - "@vitest/browser-preview": "4.1.0", - "@vitest/browser-webdriverio": "4.1.0", - "@vitest/ui": "4.1.0", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "@edge-runtime/vm": { @@ -14484,6 +14306,12 @@ "@vitest/browser-webdriverio": { "optional": true }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, "@vitest/ui": { "optional": true }, @@ -14620,18 +14448,18 @@ "license": "MIT" }, "node_modules/vscode-languageclient/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/vscode-languageclient/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -15157,9 +14985,9 @@ } }, "node_modules/y-websocket/node_modules/ws": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.3.tgz", - "integrity": "sha512-jmTjYU0j60B+vHey6TfR3Z7RD61z/hmxBS3VMSGIrroOWXQEneK1zNuotOUrGyBHQj0yrpsLHPWtigEFd13ndA==", + "version": "6.2.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.6.tgz", + "integrity": "sha512-XTrf1gv7kXoVf1hbC3PAyAiPgR8Wz1blcrYIjEsUmr08BLksT41R8KbjmS9408C2ERx7v1JDLD/BkpLEttjfKA==", "license": "MIT", "optional": true, "dependencies": { @@ -15187,15 +15015,18 @@ } }, "node_modules/yaml": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", - "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", "license": "ISC", "bin": { "yaml": "bin.mjs" }, "engines": { "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" } }, "node_modules/yargs": { diff --git a/frontend/package.json b/frontend/package.json index 5b6dd5c000..f535994371 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.798.1", + "version": "1.816.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -22,28 +22,23 @@ "test:e2e": "playwright test" }, "devDependencies": { - "@floating-ui/core": "^1.3.1", "@hey-api/openapi-ts": "^0.43.0", "@melt-ui/pp": "^0.3.2", - "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.57.0", "@sveltejs/adapter-static": "^3.0.6", - "@sveltejs/kit": "^2.53.4", + "@sveltejs/kit": "^2.69.1", "@sveltejs/package": "^2.5.7", "@sveltejs/vite-plugin-svelte": "^7.0.0", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", "@types/d3-zoom": "^3.0.3", - "@types/diff": "^7.0.1", "@types/lodash": "^4.14.195", "@types/vscode": "^1.83.5", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.60.0", - "@zerodevx/svelte-toast": "^0.9.6", "autoprefixer": "^10.4.13", "cssnano": "^6.0.1", - "d3-dag": "^0.11.5", "dts-bundle-generator": "^9.5.1", "eslint": "^8.47.0", "eslint-config-prettier": "^8.6.0", @@ -57,32 +52,27 @@ "postcss-load-config": "^4.0.1", "prettier": "^3.1.0", "prettier-plugin-svelte": "^3.3.3", - "style-to-object": "^0.4.1", "stylelint-config-recommended": "^13.0.0", - "svelte": "^5.53.5", - "svelte-awesome-color-picker": "^3.0.4", "svelte-check": "^4.4.3", "svelte-fast-check": "^0.4.5", - "svelte-floating-ui": "^1.5.8", - "svelte-highlight": "^7.6.0", - "svelte-popperjs": "^1.3.2", "svelte-preprocess": "^6.0.0", - "svelte-range-slider-pips": "^2.3.1", - "svelte-splitpanes": "^8.0.9", "tailwindcss": "^3.4.1", - "tar": "^7.5.4", + "tar": "^7.5.18", "tslib": "^2.6.1", "typescript": "^5.5.0", "vite": "^8.2.0", "vite-plugin-mkcert": "^2.0.0", - "vitest": "^4.1.0", + "vitest": "^4.1.11", "vitest-browser-svelte": "^2.0.1" }, "overrides": { "monaco-graphql": { "monaco-editor": "$monaco-editor" }, - "tar": "$tar" + "tar": "$tar", + "dompurify": "$dompurify", + "cookie": "^0.7.0", + "handlebars": "^4.7.9" }, "type": "module", "dependencies": { @@ -95,8 +85,10 @@ "@codingame/monaco-vscode-standalone-json-language-features": "=25.0.0", "@codingame/monaco-vscode-standalone-languages": "=25.0.0", "@codingame/monaco-vscode-standalone-typescript-language-features": "=25.0.0", + "@floating-ui/core": "^1.3.1", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", + "@melt-ui/svelte": "^0.86.2", "@popperjs/core": "^2.11.8", "@redocly/json-to-json-schema": "^0.0.1", "@scalar/openapi-parser": "^0.15.0", @@ -105,6 +97,8 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "@zerodevx/svelte-toast": "^0.9.6", + "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", @@ -114,10 +108,11 @@ "chartjs-adapter-date-fns": "^3.0.0", "chartjs-plugin-zoom": "^2.0.0", "clone": "^2.1.2", + "d3-dag": "^0.11.5", "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", - "diff": "^7.0.0", - "dompurify": "^3.3.1", + "diff": "^8.0.3", + "dompurify": "^3.4.13", "driver.js": "^1.3.0", "esm-env": "^1.0.0", "fast-equals": "^5.0.1", @@ -129,8 +124,8 @@ "lru-cache": "^11.1.0", "lucide-svelte": "^0.540.0", "mdast-util-find-and-replace": "^3.0.2", - "mermaid": "^11.15.0", - "minimatch": "^10.0.1", + "mermaid": "^11.16.1", + "minimatch": "^10.2.3", "modern-screenshot": "^4.7.0", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", @@ -142,15 +137,23 @@ "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", "quicktype-core": "^23.2.6", - "quill": "^1.3.7", + "quill": "^2.0.3", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "rfc4648": "^1.5.3", "runed": "^0.36.0", + "style-to-object": "^0.4.1", + "svelte": "^5.55.7", + "svelte-awesome-color-picker": "^3.0.4", "svelte-carousel": "^1.0.25", "svelte-exmarkdown": "^5.0.0", + "svelte-floating-ui": "^1.5.8", + "svelte-highlight": "^7.6.0", "svelte-infinite-loading": "^1.4.0", + "svelte-popperjs": "^1.3.2", + "svelte-range-slider-pips": "^2.3.1", + "svelte-splitpanes": "^8.0.9", "tailwind-merge": "^1.13.2", "unist-util-visit": "^5.0.0", "vscode": "npm:@codingame/monaco-vscode-extension-api@=25.0.0", @@ -177,7 +180,7 @@ "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", - "yaml": "^2.8.0", + "yaml": "^2.8.3", "yjs": "^13.6.7", "zod": "^4.1.10" }, diff --git a/frontend/src/lib/actingUser.svelte.ts b/frontend/src/lib/actingUser.svelte.ts new file mode 100644 index 0000000000..30da65ff83 --- /dev/null +++ b/frontend/src/lib/actingUser.svelte.ts @@ -0,0 +1,74 @@ +import { untrack } from 'svelte' +import { fromStore } from 'svelte/store' +import { SvelteMap } from 'svelte/reactivity' +import { userStore, workspaceStore, type UserExt } from '$lib/stores' +import { getWorkspaceRole, type RoleLookup } from '$lib/user' + +/** + * The user acting in a workspace that is not necessarily the one the top nav points at — an AI + * session or a workspace-specific variant acts on a workspace the nav deliberately is not on. + * + * `$userStore` answers for the navigation workspace at no cost, exactly as every permission check + * in the app did before this hook existed — including when it holds nobody, which reads as unknown + * and refuses. Every other workspace is looked up, and an unresolved user there is `undefined`: it + * must never fall back to the navigation user, whose rights belong to another workspace. + * `canWrite`/`isOwner` refuse for an unknown user, which is the only safe answer. A caller that + * must not render that refusal as a denial asks `resolved` first. + */ +export function useActingUser(workspace: () => string | undefined) { + const navWorkspace = fromStore(workspaceStore) + const navUser = fromStore(userStore) + const looked = new SvelteMap() + // The workspace this effect last acted on, so arriving at one is distinguishable from the + // effect re-running while already there. + let asking: string | undefined + + $effect(() => { + const ws = workspace() + if (asking !== ws) { + asking = ws + // Dropped on the way *in*, not on the way out: a lookup that fails after the acting + // workspace has already moved on has no entry to clear at the moment it is left, so + // clearing it there would keep a refusal that no attempt is behind any more. + if (ws && untrack(() => looked.get(ws)?.kind) === 'lookup_failed') looked.delete(ws) + } + if (!ws || ws === navWorkspace.current) return + // Any settled answer stops the asking, a failure included — otherwise recording one + // would re-enter this effect and loop. + if (looked.has(ws)) return + untrack(() => { + // Memoized process-wide, so two components pointed at the same workspace share one + // request rather than each issuing their own. + getWorkspaceRole(ws).then((lookup) => looked.set(ws, lookup)) + }) + }) + + function userIn(ws: string | undefined): UserExt | undefined { + if (!ws) return undefined + if (ws === navWorkspace.current) return navUser.current + const lookup = looked.get(ws) + return lookup?.kind === 'resolved' ? lookup.user : undefined + } + + return { + /** The acting user in `ws`, or `undefined` when it is not known. Only workspaces this + * hook has been pointed at are looked up; the rest read as unknown. */ + in: userIn, + /** Whether `ws` has an answer at all — a user, or a lookup that came back without one. + * The navigation workspace always has one: `$userStore`, "nobody" included. */ + resolved: (ws: string | undefined): boolean => + !!ws && (ws === navWorkspace.current || looked.has(ws)), + get current(): UserExt | undefined { + return userIn(workspace()) + }, + /** Drop the lookups that came back empty so they are asked again. Arriving at a + * workspace already does this; a long-lived editor must call this too when it starts a + * fresh session on the workspace it is already on, or a `whoami` that happened to fail + * pins it to "unknown user" for as long as it stays there. */ + forgetFailures(): void { + for (const [ws, lookup] of looked) { + if (lookup.kind === 'lookup_failed') looked.delete(ws) + } + } + } +} diff --git a/frontend/src/lib/aiStore.test.ts b/frontend/src/lib/aiStore.test.ts index fed6147b62..be8980e93a 100644 --- a/frontend/src/lib/aiStore.test.ts +++ b/frontend/src/lib/aiStore.test.ts @@ -35,6 +35,24 @@ describe('setCopilotInfo legacy /thinking migration', () => { expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) }) + it('keeps the models but turns the assistant off when the workspace disabled it', () => { + setCopilotInfo({ + providers: { + anthropic: { + resource_path: 'u/admin/anthropic', + models: ['claude-sonnet-4-6'] + } + }, + copilot_disabled: true + }) + + const info = get(copilotInfo) + expect(info.enabled).toBe(false) + expect(info.workspaceDisabled).toBe(true) + // The providers still describe what AI agent steps can run on. + expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) + }) + it('defaults provider web search on unless explicitly disabled', () => { setCopilotInfo({ providers: { diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 0955f68ee5..ff02157e4a 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -7,6 +7,7 @@ import { type AIProviderModel, type AIProvider, type AIConfig, + type FreeTierInfo, type ModelPriceOverride } from './gen' import { @@ -40,23 +41,35 @@ export const copilotSessionModel = writable( export const copilotInfo = writable<{ enabled: boolean + // The workspace hid the assistant (`ai_config.copilot_disabled`). `enabled` is then false + // whatever the providers say, and the AI entry points that nudge "configure AI" when + // `enabled` is off render nothing at all instead. + workspaceDisabled: boolean codeCompletionModel?: AIProviderModel defaultModel?: AIProviderModel metadataModel?: AIProviderModel aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Context windows per `provider:model`, overriding the built-in window table. */ + contextWindowPerModel?: Record /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> + // Set only when the workspace has no AI provider of its own and is running on + // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but + // that is a different state from "never configured" and the UI must say so. + freeTier?: FreeTierInfo }>({ enabled: false, + workspaceDisabled: false, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + contextWindowPerModel: {}, modelPricing: {}, webSearchEnabledProviders: {} }) @@ -66,7 +79,7 @@ export const copilotInfo = writable<{ aiUserDisabled.subscribe((disabled) => { copilotInfo.update((info) => ({ ...info, - enabled: info.aiModels.length > 0 && !disabled + enabled: info.aiModels.length > 0 && !disabled && !info.workspaceDisabled })) }) @@ -121,9 +134,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { return model }) + const workspaceDisabled = aiConfig.copilot_disabled === true copilotInfo.set({ - // Providers are configured; the per-user opt-out is the only thing that can gate it off. - enabled: !get(aiUserDisabled), + // Providers are configured; only the workspace or per-user opt-outs can gate it off. + enabled: !workspaceDisabled && !get(aiUserDisabled), + workspaceDisabled, // Strip the deprecated /thinking suffix from the configured model slots too, // otherwise a workspace whose default still carries it sends an invalid model id. codeCompletionModel: stripModelSuffix(aiConfig.code_completion_model), @@ -132,22 +147,29 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + contextWindowPerModel: aiConfig.context_window_per_model ?? {}, + webSearchEnabledProviders, modelPricing: aiConfig.model_pricing ?? {}, - webSearchEnabledProviders + freeTier: aiConfig.free_tier }) } else { copilotSessionModel.set(undefined) copilotInfo.set({ enabled: false, + workspaceDisabled: aiConfig.copilot_disabled === true, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + contextWindowPerModel: {}, + webSearchEnabledProviders: {}, modelPricing: {}, - webSearchEnabledProviders: {} + // An exhausted free grant lands here — no providers, but the reason AI is off + // is "you used it up", not "you never set it up". + freeTier: aiConfig.free_tier }) } } diff --git a/frontend/src/lib/assets/app.css b/frontend/src/lib/assets/app.css index 760f4c0261..4217c95831 100644 --- a/frontend/src/lib/assets/app.css +++ b/frontend/src/lib/assets/app.css @@ -3,7 +3,10 @@ @tailwind components; @tailwind utilities; -@media (min-width: 1760px) { +/* Keyed on the screen, not the window: a viewport query rescales the whole app + whenever the window is resized or split. Mirrored in editorFontSize.svelte.ts, + TextInput.svelte's leading classes and FlowGraphPreviewButton.svelte. */ +@media (min-device-width: 1760px) { :root { font-size: 18px; } @@ -209,33 +212,24 @@ U+1fac6, U+1fae0-1fae6, U+1fae8-1faea, U+1faef-1faf8; } - .prose-xs ul { - margin-top: 0.5rem; - list-style-type: '- '; - padding-left: 1.5rem; - } - + /* Bullets read as a dash rather than a disc. Only the glyph is overridden: + indentation and vertical rhythm stay with Tailwind Typography so ordered + and unordered lists line up with each other. */ .prose ul { - margin-top: 1.5rem; list-style-type: '- '; - padding-left: 3rem; } - /* The '- ' list markers, horizontal rules and blockquote bars otherwise - fall through to Tailwind Typography's default bullet/border colors, which - are nearly invisible on dark backgrounds (e.g. the AI chat). Use - theme-aware tokens so they stay readable in both light and dark mode. */ - .prose-xs ul > li::marker, - .prose ul > li::marker { + /* List markers, horizontal rules and blockquote bars take the tertiary/light + tokens the typography config maps them to, which is too faint to read on + the denser markdown surfaces (e.g. the AI chat). Step them up one. */ + .prose :is(ul, ol) > li::marker { color: rgb(var(--color-text-secondary)); } - .prose-xs hr, .prose hr { border-top-color: rgb(var(--color-border-normal)); } - .prose-xs blockquote, .prose blockquote { border-left-color: rgb(var(--color-border-normal)); } diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/cloud.ts b/frontend/src/lib/cloud.ts index dbf4a58f48..30b2e54b75 100644 --- a/frontend/src/lib/cloud.ts +++ b/frontend/src/lib/cloud.ts @@ -6,6 +6,15 @@ export function isCloudHosted(): boolean { // may be missing or a stub with no `location`. Same defensive shape as // `isChromiumBrowser`. if (!BROWSER) return false + // Dev only: the cloud-specific UI (quotas, plan upgrade, the pre-approved trial offer) + // is otherwise unreachable from localhost. `localStorage.cloudHostedOverride = '1'` opts + // a browser in against a backend started with CLOUD_HOSTED. + if ( + import.meta.env.DEV && + globalThis.window?.localStorage?.getItem('cloudHostedOverride') === '1' + ) { + return true + } return globalThis.window?.location?.hostname == 'app.windmill.dev' } diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index 259d9c1cf9..9eb019e838 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -23,6 +23,8 @@ export interface SchemaProperty { pattern?: string default?: any enum?: EnumType + /** Display names by stored value, for an enum's options or a one-of's variants. */ + enumLabels?: Record contentEncoding?: 'base64' | 'binary' format?: string items?: { @@ -35,6 +37,8 @@ export interface SchemaProperty { } min?: number max?: number + /** Height a string field's text area opens at, in rows. */ + minRows?: number currency?: string currencyLocale?: string multiselect?: boolean @@ -43,6 +47,8 @@ export interface SchemaProperty { required?: string[] showExpr?: string hideWhenChatEnabled?: boolean + /** Why the oneOf variant is chat mode's to pick. Set = selector disabled, reason shown. */ + lockOneOfWhenChatEnabled?: string password?: boolean order?: string[] nullable?: boolean diff --git a/frontend/src/lib/components/AIAgentLogViewer.svelte b/frontend/src/lib/components/AIAgentLogViewer.svelte deleted file mode 100644 index 2549fbb29f..0000000000 --- a/frontend/src/lib/components/AIAgentLogViewer.svelte +++ /dev/null @@ -1,281 +0,0 @@ - - -{#if job} -
- {}} - mode="aiagent" - /> -
-{/if} diff --git a/frontend/src/lib/components/AIProviderPicker.svelte b/frontend/src/lib/components/AIProviderPicker.svelte index 4392a2d513..f69bb6e8f4 100644 --- a/frontend/src/lib/components/AIProviderPicker.svelte +++ b/frontend/src/lib/components/AIProviderPicker.svelte @@ -3,23 +3,32 @@ import Select from './select/Select.svelte' import { fetchAvailableModels, AI_PROVIDERS } from './copilot/lib' import type { AIProvider, ProviderConfig } from '$lib/gen' - import { workspaceStore } from '$lib/stores' - import { get } from 'svelte/store' - import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' - import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import ResourcePicker from './ResourcePicker.svelte' - import ToggleButtonMore from './common/toggleButton-v2/ToggleButtonMore.svelte' import Toggle from './Toggle.svelte' import { saveConfig, removeConfig, isSameAsStoredConfig } from './aiProviderStorage' import AIReasoningEffortPicker from './AIReasoningEffortPicker.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { value: ProviderConfig | undefined disabled?: boolean actions?: Snippet + /** The workspace the surface operates on, which a session or fork editor sets to something + * other than the one being navigated. Resources and the models read off them are per + * workspace, so without it this offers what the wrong one holds. */ + workspace?: string | undefined } - let { value: _uncheckedValue = $bindable(), disabled = false, actions }: Props = $props() + let { + value: _uncheckedValue = $bindable(), + disabled = false, + actions, + workspace = undefined + }: Props = $props() + + let effectiveWorkspace = $derived(workspace ?? $operatingWorkspace ?? '') let value = $derived.by(() => { if (!_uncheckedValue || typeof _uncheckedValue !== 'object') return undefined @@ -30,7 +39,13 @@ let availableModels = $state([]) let filterText = $state('') - let modelsCache = new Map() + // Keyed by provider *and* path: two `customai` resources point at different base URLs, so they + // do not share a model list. + let modelsCache = new Map() + + // The resource picker offers every provider type at once and the pick is what names the kind. + // One string for the component's life: it is what the picker queries with. + const providerResourceTypes = Object.keys(AI_PROVIDERS).join(',') if (!_uncheckedValue) { _uncheckedValue = { @@ -57,12 +72,6 @@ return r }) - // Provider options for the toggle button group - const providerOptions = Object.entries(AI_PROVIDERS).map(([key, details]) => ({ - value: key as AIProvider, - label: details.label - })) - async function loadModels(signal?: AbortSignal) { const provider = value?.kind const resourceValue = value?.resource @@ -73,20 +82,20 @@ } loading = true - if (modelsCache.has(provider)) { - availableModels = modelsCache.get(provider) || [] + const cacheKey = `${effectiveWorkspace}:${provider}:${resourcePath}` + if (modelsCache.has(cacheKey)) { + availableModels = modelsCache.get(cacheKey) || [] loading = false return } try { - const workspace = get(workspaceStore) || '' - const models = await fetchAvailableModels(resourcePath, workspace, provider, signal) + const models = await fetchAvailableModels(resourcePath, effectiveWorkspace, provider, signal) if (signal?.aborted) { return } availableModels = models - modelsCache.set(provider, models) + modelsCache.set(cacheKey, models) } catch (e) { if (signal?.aborted) { return @@ -101,15 +110,24 @@ } } - // Handle provider selection - function onProviderChange(selectedProvider: AIProvider) { - if (value) { - value.kind = selectedProvider - value.resource = '' - value.model = '' - // Reasoning effort is model-specific; reset it with the model. - value.reasoning_effort = undefined + /** + * The provider kind follows the resource that was picked. Driven by the pick rather than by an + * effect on the picker's `valueType`, which also resolves for the value the field was opened on + * and would rewrite a saved config just for being looked at. + */ + function onResourcePicked(_path: string | undefined, type: string | undefined) { + // An empty type is the placeholder the picker keeps for a saved path it could not find. It + // says nothing about the provider, so the kind stands. + if (!value || !type || !(type in AI_PROVIDERS)) { + return } + if (value.kind === type) { + return + } + value.kind = type as AIProvider + // Models are per provider, and a reasoning token is per model. + value.model = '' + value.reasoning_effort = undefined } // Helper functions to handle $res: prefix like ObjectResourceInput does @@ -165,97 +183,74 @@ }) -
- - - {#snippet children({ item })} - {#each providerOptions.slice(0, 3) as option} - - {/each} - p.value === value?.kind) >= 3 ? '' : 'More'} - togglableItems={providerOptions.slice(3)} - {item} - bind:selected={() => value?.kind, (v) => v && onProviderChange(v)} - /> - {/snippet} - - - -
-
-

resource

- resourceValueToPath(value?.resource), - (v) => { - if (value) { - value.resource = pathToResourceValue(v) ?? '' - } +
+
+ Resource + + resourceValueToPath(value?.resource), + (v) => { + if (value) { + value.resource = pathToResourceValue(v) ?? '' } } - resourceType={value?.kind} - disabled={disabled || !value?.kind} - placeholder="Select resource" - selectFirst={true} - /> -
+ } + resourceType={providerResourceTypes} + {disabled} + {workspace} + placeholder="Select an AI provider resource" + selectFirst={false} + onValueChange={onResourcePicked} + /> +
- +
+ Model + value?.model, (v) => value && (value.model = v ?? '')} - placeholder="Select model" - disabled={disabled || !value?.kind || !resourceValueToPath(value?.resource)} - onCreateItem={(r) => { - availableModels.push(r) - if (value) value.model = r - }} - createText="Press enter to use custom model" - {loading} - clearable={false} - noItemsMsg={'No models available'} - bind:filterText + Reasoning effort + value?.reasoning_effort, (v) => value && (value.reasoning_effort = v)} + providerConfig={value} + {disabled} />
+ {/if} - - {#if value?.model} -
-

reasoning effort

- value?.reasoning_effort, (v) => value && (value.reasoning_effort = v)} - providerConfig={value} - {disabled} - /> -
- {/if} - - -
- { - if (!e.detail) { - removeConfig() - } else { - saveConfig(value) - } - }} - /> -
+
+ { + if (!e.detail) { + removeConfig() + } else { + saveConfig(value) + } + }} + />
{@render actions?.()} diff --git a/frontend/src/lib/components/AIReasoningEffortPicker.svelte b/frontend/src/lib/components/AIReasoningEffortPicker.svelte index 3ebf94c9bf..39b4e80011 100644 --- a/frontend/src/lib/components/AIReasoningEffortPicker.svelte +++ b/frontend/src/lib/components/AIReasoningEffortPicker.svelte @@ -27,7 +27,7 @@ let capability = $derived( provider && model ? getReasoningCapability(provider, model) - : { supported: false, levels: [], canDisable: false } + : { supported: false, levels: [], canDisable: false, known: false } ) // The token that turns reasoning off on a model that reasons by default diff --git a/frontend/src/lib/components/AgentResultDisplay.svelte b/frontend/src/lib/components/AgentResultDisplay.svelte new file mode 100644 index 0000000000..0388ab0845 --- /dev/null +++ b/frontend/src/lib/components/AgentResultDisplay.svelte @@ -0,0 +1,116 @@ + + +
+ {#if trace.length > 0} + + {/if} + {#if reasoning} + + (reasoningExpanded = !reasoningExpanded)} + contentClass="font-main" + > + + + {/if} + {#if trace.length > 0 || reasoning} + + Output + + {/if} +
+ {#if textOutput !== undefined} + {#if textOutput === ''} + The agent returned no answer + {:else} + + + {/if} + {:else} + {@render structuredOutput(result.output)} + {/if} + {#if answer.sources} +
+ +
+ {/if} +
+ + +
+ {#if summary.toolCalls > 0} + + {summary.toolCalls} + {summary.toolCalls === 1 ? 'tool call' : 'tool calls'} + + {/if} + {#if summary.webSearches > 0} + + {summary.webSearches} + {summary.webSearches === 1 ? 'web search' : 'web searches'} + + {/if} + {#if summary.tokens !== undefined} + {formatTokenCount(summary.tokens)} tokens + {/if} + {#if summary.cachedTokens} + {formatTokenCount(summary.cachedTokens)} cached + {/if} +
+
diff --git a/frontend/src/lib/components/AgentStreamDisplay.svelte b/frontend/src/lib/components/AgentStreamDisplay.svelte new file mode 100644 index 0000000000..cf1646a614 --- /dev/null +++ b/frontend/src/lib/components/AgentStreamDisplay.svelte @@ -0,0 +1,111 @@ + + + +
+ {#each stream.entries as entry, index (entry.kind === 'tool' ? entry.callId : index)} + {#if entry.kind === 'tool'} + {}} + labelClass={entry.success === false ? 'text-red-500' : ''} + /> + {:else} +
+ +
+ {/if} + {/each} + + {#if stream.current !== ''} +
+ + +
+ {:else if stream.reasoning !== ''} + +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/AgentTrace.svelte b/frontend/src/lib/components/AgentTrace.svelte new file mode 100644 index 0000000000..91b87fea67 --- /dev/null +++ b/frontend/src/lib/components/AgentTrace.svelte @@ -0,0 +1,137 @@ + + +
diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index e4ee3ca7b9..2255817c5e 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -1,7 +1,6 @@ - - diff --git a/frontend/src/lib/components/ArgInfo.svelte b/frontend/src/lib/components/ArgInfo.svelte index a1cfdf6276..b071434f9d 100644 --- a/frontend/src/lib/components/ArgInfo.svelte +++ b/frontend/src/lib/components/ArgInfo.svelte @@ -1,12 +1,14 @@ @@ -20,7 +22,7 @@ Setup the wmill cli for this workspace & remote:
diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 35bd88abec..4775409252 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,19 +1,26 @@ - +
- {#if dbSelector} - {@render dbSelector()} - {/if} - {#if dbSupportsSchemas && !multiSelectMode} - e.stopPropagation()} - onchange={() => toggleSchemaSelection(schemaKey)} - /> - - {/if} - {schemaKey} - - {schemaTables.length} - - - -
- - {#each schemaTables as tableKey} - {@const isDisabled = isTableDisabled(schemaKey, tableKey)} - {@const isChecked = isTableSelected(schemaKey, tableKey) || isDisabled} - {@const isCurrentPreview = - selected.schemaKey === schemaKey && selected.tableKey === tableKey} -
{ - selected.schemaKey = schemaKey - selected.tableKey = tableKey - toggleTableSelection(schemaKey, tableKey) - }} - onkeydown={(e) => { - if (e.key === 'Enter' || e.key === ' ') { - selected.schemaKey = schemaKey - selected.tableKey = tableKey - toggleTableSelection(schemaKey, tableKey) - } - }} - > - - e.stopPropagation()} - onchange={() => toggleTableSelection(schemaKey, tableKey)} - /> - - -

{tableKey}

- + {#if dtOpen} + {#if root.error} +

{root.error}

+ {/if} + {#each root.schemas as sc (sc.schemaKey)} + {@const schemaOpen = isExpanded(root.datatable, sc.schemaKey)} + {@const indent = root.datatable !== undefined ? 'pl-7' : 'pl-3'} + {#if dbSupportsSchemas} -
- {/each} - - - {/each} - {:else} - - {#each filteredTableKeys as tableKey} - - - {/each} - {/if} + ]} + btnId={'db-manager-schema-actions-' + onlyAlphaNumAndUnderscore(sc.schemaKey)} + /> + {/if} +
+ + {/if} + + + {#if schemaOpen || !dbSupportsSchemas} + {@const tableIndent = dbSupportsSchemas + ? root.datatable !== undefined + ? 'pl-11' + : 'pl-7' + : root.datatable !== undefined + ? 'pl-7' + : 'pl-3'} + {#each sc.tables as tableKey (tableKey)} + {@const entry = { + datatable: root.datatable, + schema: sc.schemaKey, + table: tableKey + }} + {@const hasMenu = !multiSelectMode} + {@const isSelected = + root.datatable === currentDatatable && + selected.schemaKey === sc.schemaKey && + selected.tableKey === tableKey} + + {/each} + {#if canCreateTableIn(root.datatable, sc.schemaKey)} + + {/if} + {/if} + + {/each} + {#if dbSupportsSchemas && search.trim() === '' && canCreateSchemaIn(root.datatable)} + + {/if} + {/if} + {/each}
- {#if !multiSelectMode} - - {/if} - {#if tableKey && colDefs?.[tableKey]?.length} - {@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey })} - + {#if mainPane} + {@render mainPane()} + {:else if tableKey && colDefs?.[tableKey]?.length} + {@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey, whereClause })} + (rowFilter = undefined)} + bind:this={_dbTable} + /> {:else if databaseIsEmpty}
Database is empty @@ -557,7 +1041,7 @@ Import schema from database - {#if !!$userStore?.is_admin || !!$superadmin} + {#if !!actingUser?.is_admin || !!$superadmin} {/snippet} diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index 68ba0c5f30..d4e00ce30b 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -1,16 +1,21 @@ + + + (open = false) }} +> + { + // The row underneath folds on click, and picking a role is not that. + e.stopPropagation() + open = !open + }} + > + + {role} + + + anchorEl!.getBoundingClientRect())} + onSelectValue={(item) => { + open = false + if (item.value !== role) onSelect(item.value) + }} + /> + diff --git a/frontend/src/lib/components/DbForeignKeyTooltip.svelte b/frontend/src/lib/components/DbForeignKeyTooltip.svelte new file mode 100644 index 0000000000..526d6b2b69 --- /dev/null +++ b/frontend/src/lib/components/DbForeignKeyTooltip.svelte @@ -0,0 +1,23 @@ + + +
+
+ References {targetTable}.{targetColumn} +
+ +
diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index b6833521f8..0dab262fd0 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -3,14 +3,25 @@ import Modal2 from './common/modal/Modal2.svelte' import NewDataTableMigrationModal from './workspaceSettings/NewDataTableMigrationModal.svelte' import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' - import { splitSqlStatements, isDdlStatement } from './sqlDdl' + import { joinSqlStatements, splitSqlRuns } from './sqlDdl' + import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' + import { withMigrationRole } from './datatableMigrationRole' - let { workspace, datatable }: { workspace: string; datatable: string } = $props() + let { + workspace, + datatable, + role + }: { + workspace: string + datatable: string + /** The role the editor runs as. The migration declares it, or it would run as admin. */ + role?: string + } = $props() type Choice = 'run' | 'migrate' | 'cancel' - let promptStatement = $state(undefined) + let promptStatements = $state([]) let promptOpen = $state(false) let resolvePrompt: ((choice: Choice) => void) | undefined = undefined let resolveMigrationClosed: ((created: boolean) => void) | undefined = undefined @@ -22,11 +33,15 @@ // toast action after a migration is created here. let migrationsModal = $state(undefined) + // The block is shown as-is in the prompt and becomes the migration body, where + // every statement inside the BEGIN; ... END; frame must be `;`-terminated. + let promptSql = $derived(joinSqlStatements(promptStatements)) + function finishPrompt(choice: Choice) { const r = resolvePrompt resolvePrompt = undefined promptOpen = false - promptStatement = undefined + promptStatements = [] r?.(choice) } @@ -48,10 +63,10 @@ } } - function promptDdl(statement: string): Promise { + function promptDdl(statements: string[]): Promise { return new Promise((resolve) => { resolvePrompt = resolve - promptStatement = statement + promptStatements = statements promptOpen = true }) } @@ -65,61 +80,66 @@ // Open the prefilled new-migration modal. Resolves with whether a migration // was actually created (false if the user cancelled / closed it). - function openMigrationModal(statement: string): Promise { + function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: statement }) + newMigrationModal?.open({ codeUp: withMigrationRole(sql, role) }) }) } /** - * Inspect `code` for DDL statements. For each one, prompt the user to run it - * anyway or turn it into a migration (prompts shown one at a time). Returns - * whether to proceed and the code to run (with migrated statements stripped). + * Inspect `code` for DDL statements. Each run of adjacent DDL statements is + * prompted for once (runs shown one at a time) and becomes a single migration, + * so a chain of schema changes applies in one transaction instead of asking + * once per statement. Returns whether to proceed and the code to run (with + * migrated statements stripped). */ export async function guard( code: string ): Promise<{ proceed: boolean; code: string; ranMigration: boolean }> { migrationRan = false - const statements = splitSqlStatements(code) - if (!statements.some((s) => isDdlStatement(s))) { + const runs = splitSqlRuns(code) + if (!runs.some((r) => r.isDdl)) { return { proceed: true, code, ranMigration: false } } const kept: string[] = [] - for (const statement of statements) { - if (!isDdlStatement(statement)) { - kept.push(statement) + for (const run of runs) { + if (!run.isDdl) { + kept.push(...run.statements) continue } - // Re-prompt for this statement until the user makes a terminal choice; + // Re-prompt for this run until the user makes a terminal choice; // cancelling the migration modal returns to the prompt with the DDL intact. for (;;) { - const choice = await promptDdl(statement) + const choice = await promptDdl(run.statements) if (choice === 'cancel') { + logDdlGuardChoice('cancelled') return { proceed: false, code, ranMigration: migrationRan } } if (choice === 'run') { - kept.push(statement) + logDdlGuardChoice('run_anyway') + kept.push(...run.statements) break } - // migrate: only strip the statement once a migration is actually + // migrate: only strip the statements once a migration is actually // created; if the modal was cancelled, loop back to the prompt. - const created = await openMigrationModal(statement) + const created = await openMigrationModal(joinSqlStatements(run.statements)) if (created) { + logDdlGuardChoice('migrated') break } } } - return { proceed: true, code: kept.join(';\n'), ranMigration: migrationRan } + return { proceed: true, code: joinSqlStatements(kept), ranMigration: migrationRan } } 1 ? 'Schema changes detected' : 'Schema change detected'} fixedWidth="md" fixedHeight="adaptive" bind:isOpen={promptOpen} @@ -127,12 +147,22 @@ >

- This looks like a schema-changing (DDL) statement. Schema changes are best tracked as - migrations rather than run ad-hoc. Create a migration for it instead? + {#if promptStatements.length > 1} + These {promptStatements.length} consecutive statements are schema-changing (DDL). Schema changes + are best tracked as migrations rather than run ad-hoc. Create a single migration for them instead? + {:else} + This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? + {/if}

+ {#if role} +

+ It will run as role {role}. +

+ {/if}
{promptStatement ?? ''}
{promptSql}
diff --git a/frontend/src/lib/components/DefaultScripts.svelte b/frontend/src/lib/components/DefaultScripts.svelte index dde49ecf2f..b5b83c7067 100644 --- a/frontend/src/lib/components/DefaultScripts.svelte +++ b/frontend/src/lib/components/DefaultScripts.svelte @@ -1,10 +1,13 @@ -{#if $userStore?.is_admin || $userStore?.is_super_admin} +{#if actingUser?.is_admin || actingUser?.is_super_admin} diff --git a/frontend/src/lib/components/DefaultScriptsInner.svelte b/frontend/src/lib/components/DefaultScriptsInner.svelte index eaa3d989a0..b062fb4c49 100644 --- a/frontend/src/lib/components/DefaultScriptsInner.svelte +++ b/frontend/src/lib/components/DefaultScriptsInner.svelte @@ -1,10 +1,13 @@ + + +
+
+ It was saved from another tab or session since this one read it, so your changes here are no + longer being saved. +
+
+ + +
+
+
diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 6738e7ace8..5061636835 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -145,6 +145,9 @@ } } async function getMenuElements(): Promise { + // Runs on every pointerdown anywhere, for every mounted dropdown. Skip the + // whole-document query when the outside handler below cannot act anyway. + if (!usePointerDownOutside || !open) return [] return Array.from(document.querySelectorAll('[data-menu]')) as HTMLElement[] } diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 5f8b96c32f..7749310b5b 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -53,12 +53,17 @@ {#if item.icon} {/if} -

- {item.displayName} -

+
+

+ {item.displayName} +

+ {#if item.description} +

{item.description}

+ {/if} +
{@render item.extra?.()} {#if item.shortcut || item.selected || item.toggle !== undefined} @@ -605,7 +611,7 @@ >
{argName} - {#if !uiOnly} + {#if !uiOnly && !locked}
{#snippet trigger()} @@ -654,7 +660,7 @@ Required {/if} - {#if !uiOnly} + {#if !uiOnly && !locked} diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 5078e0a83b..6c89f1a039 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -9,6 +9,11 @@ type: 'string' | 'number' | 'boolean' allowMultiple?: boolean format?: 'json' + /** Boolean only: the value the filter holds while unset (defaults to false). + * Selecting a filter whose default is false sets it to true immediately rather + * than opening a true/false picker whose only useful choice is true. A default-true + * boolean still opens the picker, since choosing false is the meaningful action. */ + default?: boolean } | { type: 'date' @@ -107,30 +112,68 @@ } /** - * Creates a URL-synced filter instance that automatically syncs with URL search parameters + * Creates a URL-synced filter instance that automatically syncs with URL search parameters. + * + * `initial` supplies defaults for keys the URL doesn't carry (a route segment, a persisted + * toggle); the returned `seed` re-applies them after a navigation has rewritten the query. */ export function useUrlSyncedFilterInstance( - schemaRec: T - ): { val: Partial> } { + schemaRec: T, + initial?: Partial> + ): { + val: Partial> + seed: (values: Partial>) => void + } { // Build the Zod schema from the filter schema const zodSchema = filterSchemaRecToZodSchema(schemaRec) // Create URL-synced search params const urlFilter = useSearchParams(zodSchema) as Record + // A default has to arrive as a URL param: the URL→instance effect below drops whatever the + // URL lacks, so a value written to the instance is undone on the next sync. Going through + // urlFilter rather than straight to history keeps the search-param cells in step, so it + // does not matter whether a popstate follows. + function seed(values: Partial>) { + const sp = new URLSearchParams(window.location.search) + for (const [key, value] of Object.entries(values) as [string, unknown][]) { + if (value === undefined || value === null || sp.has(key)) continue + urlFilter[key] = value instanceof Date ? value.toISOString() : value + } + } + if (initial) seed(initial) + // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) - // Sync URL params to filter instance on initialization and when URL changes + // Sync URL params to filter instance, reactively. Reading urlFilter[key] tracked + // means browser Back/Forward — which mutates useSearchParams' cells on popstate — + // flows into the instance (chips, kind toggle, results), not just the first render. + // The write happens untracked so it can't self-trigger, and the equality check plus + // the reverse effect's own guard keep the two directions from ping-ponging. for (const key of Object.keys(schemaRec)) { - let urlValue = urlFilter[key] - if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { - const d = new Date(urlValue) - urlValue = isNaN(d.getTime()) ? null : d - } - if (urlValue !== undefined && urlValue !== null) { - ;(filterInstance.val as any)[key] = urlValue - } + $effect(() => { + let urlValue = urlFilter[key] + if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { + const d = new Date(urlValue) + urlValue = isNaN(d.getTime()) ? null : d + } + untrack(() => { + const current = (filterInstance.val as any)[key] + const same = + urlValue instanceof Date && current instanceof Date + ? urlValue.getTime() === current.getTime() + : current === (urlValue ?? undefined) + if (same) return + if (urlValue !== undefined && urlValue !== null) { + ;(filterInstance.val as any)[key] = urlValue + } else if (current !== undefined) { + // Key dropped from the URL (Back to a state without it): clear it so a + // stale chip / filter doesn't linger against the navigated-to URL. + delete (filterInstance.val as any)[key] + } + }) + }) } // Sync filter instance changes back to URL params @@ -150,7 +193,15 @@ }) } - return filterInstance + return { + get val() { + return filterInstance.val + }, + set val(v: Partial>) { + filterInstance.val = v + }, + seed + } } function filterToText(filter: FilterInstance, schema: F): string { @@ -275,6 +326,17 @@ class?: string placeholder?: string autofocus?: boolean + // Applied as the id of the underlying editable, so a parent can focus it or recognise its + // key events by id (the searchbar is a contenteditable, not an ). + inputId?: string + // Free-text mode: while the input holds only free text (no specific filter tag is + // being edited and no non-default filter is set), suppress the suggestions dropdown + // so it behaves like a plain search box. This frees the arrow keys for the + // surrounding UI (e.g. a results list). The dropdown returns the moment a specific + // filter is present (e.g. `path: u/me/abc`). + hideDropdownOnFreeText?: boolean + // Notified whenever the dropdown's effective visibility changes + onDropdownVisibleChange?: (visible: boolean) => void } type SchemaT = FilterSchemaRec // TODO: Generic @@ -284,7 +346,10 @@ presets: _presets = [], class: className, placeholder = 'Filter...', - autofocus + autofocus, + hideDropdownOnFreeText = false, + onDropdownVisibleChange, + inputId }: Props = $props() let _value = new DebouncedTempValue( @@ -298,6 +363,24 @@ let currentTag: keyof SchemaT | undefined = $state() let currentTextSegment = $state({ text: '', start: 0, end: 0 }) let open = $state(false) + + // A specific filter is in play when a tag is being edited or any non-free-text filter + // is set. + let hasSpecificFilter = $derived( + !!currentTag || Object.keys(value).some((k) => k !== '_default_') + ) + // A plain search term is being typed (free text, no specific filter). + let hasFreeText = $derived(!!String(value['_default_'] ?? '').trim()) + // Effective dropdown visibility. Free-text mode suppresses the dropdown ONLY while the + // user is typing a plain search term: it still opens when the input is empty (so the + // available filters stay discoverable) and whenever a specific filter is set or being + // edited. That leaves the arrow keys for the surrounding list only during free-text search. + let dropdownVisible = $derived( + open && (!hideDropdownOnFreeText || hasSpecificFilter || !hasFreeText) + ) + $effect(() => { + onDropdownVisibleChange?.(dropdownVisible) + }) let inputElement: HTMLDivElement | undefined = $state() let highlightedIndex = $state(0) let taggedTextInput: TaggedTextInput | undefined = $state() @@ -347,9 +430,17 @@ key, filterSchema, onClick: () => { - // Replace the text segment with the new filter tag const before = asText.val.slice(0, currentTextSegment.start) const after = asText.val.slice(currentTextSegment.end) + if (schema[key].type === 'boolean' && schema[key].default !== true) { + // Set the only useful value and reparse to canonical text. The space is + // required: dropping the segment must not fuse the tags that flanked it. + asText.val = `${before} ${after}` + value[key] = true as any + asText.reparse() + return + } + // Replace the text segment with the new (empty) filter tag; the value picker opens. asText.val = `${before}${before && !before.endsWith(' ') ? ' ' : ''}${key}:\\\u00A0${after}`.trim() + '\u00A0' @@ -406,6 +497,28 @@ onClick: () => setValueForCurrentTag(false) } ] + } else if (filter.type === 'string' && filter.format !== 'json') { + // A plain string filter has no fixed options, but any presets targeting this tag + // (`:`) are exactly its useful values — surface them as suggestions so + // picking one is a click, matching the top-level preset row. Unescape the tagged + // syntax's `\ ` back to a real space for the stored value. + const prefix = `${String(currentTag)}:` + const suffix = String(value[currentTag!] ?? '') + .trim() + .toLowerCase() + return _presets + .filter((p) => p.value.startsWith(prefix) && !asText.val.includes(p.value)) + .map((p) => { + const raw = p.value.slice(prefix.length).replace(/^\\ /, '').replace(/\\ /g, ' ') + return { name: p.name, raw } + }) + .filter((p) => !suffix || p.raw.toLowerCase().includes(suffix)) + .map((p) => ({ + type: 'option' as const, + option: { value: p.raw, label: p.name }, + onClick: () => appendOrSetValueForCurrentTag(p.raw), + onNegativeClick: undefined + })) } } return [] @@ -514,7 +627,9 @@ } function handleKeyDown(e: KeyboardEvent) { - if (!open) return + // In free-text mode the dropdown is hidden; let arrow/enter keys pass through to + // the surrounding UI (e.g. list navigation) rather than steering a hidden menu. + if (!dropdownVisible) return if (e.key === 'Escape') { open = false return @@ -601,6 +716,7 @@ > (open = true)} + onKeyDown={(e) => { + // In free-text mode the searchbar coexists with a list that owns Arrow/Enter, so opening + // the dropdown on a bare navigation key would steal them from an empty box. Typing, click, + // or an already-open dropdown still open/keep it. Other searchbars keep opening on any key. + if ( + !hideDropdownOnFreeText || + !['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight', 'Enter', 'Escape', 'Tab'].includes( + e.key + ) + ) { + open = true + } + }} {autofocus} /> {#if asText.val} @@ -630,9 +758,10 @@
inputElement?.getBoundingClientRect() ?? new DOMRect()} - innerClass="!max-h-[30rem]" + innerClass="!max-h-[25rem]" strictWidth > @@ -747,6 +876,20 @@ class="border border-border-light rounded min-h-[4rem]" />
+ {:else if filter.type === 'string'} + {#if menuItems.length} +
+ {#each menuItems as item, index} + {#if item.type === 'option' && item.option} + {@render menuItem({ + onClick: item.onClick, + label: item.option.label || item.option.value, + highlighted: index === highlightedIndex + })} + {/if} + {/each} +
+ {/if} {/if} {/snippet} diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 52af60d7a7..245f60bcd0 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -10,6 +10,7 @@ type Job } from '$lib/gen' import { initHistory, redo, undo } from '$lib/history.svelte' + import { VERSION_PAGE_SIZE } from '$lib/components/diff_drawer' import { clearLinkedAgentTools, linkedAgentToolsForScope, @@ -17,13 +18,7 @@ linkedAgentToolsVersion, migrateLinkedAgentToolsScope } from '$lib/components/flows/linkedAgentToolsStore.svelte' - import { - enterpriseLicense, - userStore, - userWorkspaces, - workspaceStore, - usedTriggerKinds - } from '$lib/stores' + import { enterpriseLicense, userStore, userWorkspaces, usedTriggerKinds } from '$lib/stores' import { generateRandomString, orderedJsonStringify, @@ -41,7 +36,7 @@ import DeployOverrideConfirmationModal from '$lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte' import AIChangesWarningModal from '$lib/components/copilot/chat/flow/AIChangesWarningModal.svelte' - import { createRawSnippet, getContext, setContext, untrack } from 'svelte' + import { getContext, onDestroy, setContext, untrack } from 'svelte' import { writable } from 'svelte/store' import CenteredPage from './CenteredPage.svelte' import { Button } from './common' @@ -64,30 +59,13 @@ import { setNoteEditorContext } from './graph/noteEditor.svelte' import { GroupEditor, setGroupEditorContext } from './graph/groupEditor.svelte' import { cleanFlow } from './flows/utils.svelte' - import { - DiffIcon, - HistoryIcon, - FileJson, - Settings, - Undo, - Redo, - BookOpen, - Circle, - CheckCircle, - RefreshCw, - CheckCheck, - Disc - } from 'lucide-svelte' + import { DiffIcon, HistoryIcon, FileJson, Settings, Undo, Redo, Disc } from 'lucide-svelte' import Awareness from './Awareness.svelte' import { getAllModules } from './flows/flowExplorer' import { type FlowCopilotContext } from './copilot/flow' import { loadFlowModuleState } from './flows/flowStateUtils.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import FlowTutorials from './FlowTutorials.svelte' import FlowHistory from './flows/FlowHistory.svelte' - import { resetAllTodos, skipAllTodos } from '$lib/tutorialUtils' - import { tutorialsToDo } from '$lib/stores' - import { getTutorialIndex } from '$lib/tutorials/config' import EditorHeader from './EditorHeader.svelte' import AutosaveIndicator from './AutosaveIndicator.svelte' import type { FlowBuilderWhitelabelCustomUi } from './custom_ui' @@ -98,7 +76,18 @@ import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' import type { Trigger } from './triggers/utils' import { deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' + import { + agentDraftCanWrite, + linkedAgentPaths, + loadLinkedAgentDrafts, + type LinkedAgentDraft + } from './flows/linkedAgentDrafts' + import { agentDraftDeployRefusal } from './flows/agentDraft.svelte' + import { markAgentWritten } from './flows/agentEditorStore.svelte' + import { logReusableAgentUsage } from './flows/agentTelemetry' + import { deployDraft } from '$lib/utils_draft_deploy' + import { getUserExt } from '$lib/user' import { Triggers } from './triggers/triggers.svelte' import { StepsInputArgs } from './flows/stepsInputArgs.svelte' import { aiChatManager } from './copilot/chat/AIChatManager.svelte' @@ -116,9 +105,23 @@ import { isCloudHosted } from '$lib/cloud' import { UserDraft } from '$lib/userDraft.svelte' import { setOpenInSessionHandoff } from './sessions/openInSessionContext' + import { getEditorStoragePath, setEditorStoragePath } from './editorStoragePathContext' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) let { initialPath = $bindable(''), + /** The draft row's own path (the URL path). Unlike `initialPath`, which the + * route re-seeds from `draft_path` so the topbar shows the pending name, + * this stays where the item actually is. */ + userDraftPath = '', + onTakeLatest = undefined, pathStoreInit = undefined, newFlow, selectedId, @@ -163,9 +166,9 @@ // and the AutosaveIndicator all target it. Falls back to the global store, so // the full-page editor is unchanged; the sessions preview overrides it to the // session's (forked) workspace, so an embedded editor acts on the session's - // fork rather than the navigation workspace ($workspaceStore, which stays put). + // fork rather than the navigation workspace (`workspaceStore`, which stays put). // indicatorPath is the matching draft path. - const opWorkspace = $derived(autosaveWorkspace ?? $workspaceStore) + const opWorkspace = $derived(autosaveWorkspace ?? $operatingWorkspace) const indicatorPath = $derived(autosavePath ?? liveEditorDraftStoragePath) let initialPathStore = writable(initialPath) @@ -188,12 +191,24 @@ // Used by multiplayer deploy collision warning let deployedValue: Value | undefined = $state(undefined) // Value to diff against + let deployedLabel: string | undefined = $state(undefined) // Names it in the diff + /** The flow_version the payload in `deployedValue` came from, so the picker marks that + * one as head rather than trusting the history's first row. */ + let deployedVersionShown: number | undefined = $state(undefined) let deployedBy: string | undefined = $state(undefined) // Author let confirmCallback: () => void = $state(() => {}) // What happens when user clicks `override` in warning let open: boolean = $state(false) // Is confirmation modal open - // Draft triggers confirmation modal - let draftTriggersModalOpen = $state(false) + // Draft changes (triggers + linked agents) confirmation modal + let draftChangesModalOpen = $state(false) + /** The unsaved agent drafts the pending deploy found. Loaded rather than derived: it takes a + * request per linked agent, so it is resolved when the deploy asks. */ + let draftAgents = $state([]) + let agentCanWrite = $state>({}) + let agentRefusal = $state>({}) + + /** What the dialog's confirm hands back to `saveFlow`. */ + type DraftChangesToDeploy = { triggers: Trigger[]; agents: LinkedAgentDraft[] } // Top-bar responsive collapse. Measured via bind:clientWidth — we can't // rely on viewport `md:` because the editor lives inside other panes @@ -224,7 +239,7 @@ ] : [] ) - let confirmDeploymentCallback: (triggersToDeploy: Trigger[]) => void = () => {} + let confirmDeploymentCallback: (toDeploy: DraftChangesToDeploy) => void = () => {} // AI changes warning modal let aiChangesWarningOpen = $state(false) @@ -236,11 +251,57 @@ const job: Job | undefined = $derived(flowPreviewContent?.getJob()) let showJobStatus = $state(false) - async function handleDraftTriggersConfirmed(event: CustomEvent<{ selectedTriggers: Trigger[] }>) { - const { selectedTriggers } = event.detail + async function handleDraftChangesConfirmed( + event: CustomEvent<{ selectedTriggers: Trigger[]; selectedAgents: LinkedAgentDraft[] }> + ) { + const { selectedTriggers, selectedAgents } = event.detail // Continue with saving the flow - draftTriggersModalOpen = false - confirmDeploymentCallback(selectedTriggers) + draftChangesModalOpen = false + confirmDeploymentCallback({ triggers: selectedTriggers, agents: selectedAgents }) + } + + /** Deploy each selected agent's draft, the same way the Review & Deploy page deploys the same + * row: hand the path to `deployDraft` and let it promote whatever the draft holds, with no + * re-read to check it still matches what the dialog listed. Agents left out keep their draft + * untouched. */ + async function deployAgentDrafts(agents: LinkedAgentDraft[]) { + const ws = opWorkspace + if (!ws) return + for (const listed of draftAgents) { + // Only the rows the dialog gave a choice on. A `Read-only` or `Invalid config` agent can + // never be selected, so counting it as "kept" would record a decision the user was never + // offered and bias the pair towards keeping. + const selectable = agentCanWrite[listed.path] !== false && !agentRefusal[listed.path] + if (selectable && !agents.some((a) => a.path === listed.path)) { + logReusableAgentUsage('draft_kept_on_deploy') + } + } + for (const agent of agents) { + // Writes the resource, deletes the draft row, and clears the local hint and the workspace + // drafts cache. A failure aborts the flow save the way a failing trigger does, rather than + // deploying a flow against agents that were meant to change with it. + const deployed = await deployDraft('resource', agent.path, ws, { + draftOnly: agent.noDeployed + }) + if (!deployed.success) { + throw new Error(`Could not deploy agent ${agent.path}: ${deployed.error}`) + } + // Nothing was promoted: the draft had gone by the time the helper read it. Saying the agent + // deployed would be a lie about the one thing the toggle decides. + if (deployed.noop) { + throw new Error( + `The draft for ${agent.path} was deployed or discarded elsewhere while this deploy ran, so nothing was written for it.` + ) + } + // `deployDraft` deletes the server row but leaves any in-memory cell for this key, and that + // cell is what `agentDraftState` prefers — a still-mounted holder would otherwise keep + // feeding a phantom draft to the cards and to the next deploy dialog. Local only: `remove` + // would POST a second delete, debounced and past the baseline the first one cleared. + UserDraft.forgetLocal('resource', agent.path, { workspace: ws }) + // Every linked card and the graph key on this to refetch the agent they display. + markAgentWritten(ws, agent.path) + logReusableAgentUsage('draft_deployed_with_flow') + } } // Inside an AI session pane (SessionEditorTarget injects an aiChatManager via @@ -287,7 +348,7 @@ // a draft, else the load-time head. This catches both a concurrent deploy // (head moved since open) AND a stale draft reopened after a deploy (head == // load-time head, but the draft was forked from an older version). - const base = draftBaseVersion ?? version + const base = draftBaseVersion ?? (version != null ? String(version) : undefined) if (base === undefined) { return } @@ -298,7 +359,7 @@ path: initialPath }) - onLatest = base === flowVersion?.id + onLatest = flowVersion != null && base === String(flowVersion.id) } else { onLatest = true } @@ -414,12 +475,19 @@ } } } - async function syncWithDeployed() { + async function syncWithDeployed(opening?: number) { const flow = await FlowService.getFlowByPath({ workspace: opWorkspace!, - path: initialPath, + // The draft row's own path, not `initialPath` — the route re-seeds that from + // the draft's `draft_path` so the topbar shows the pending name, which after + // someone renames the item still names the old location. Comparing against + // that fetches the row left behind there instead of the live one. + path: userDraftPath || initialPath, withStarredInfo: true }) + // A superseded opening must not write these: the current one would then render + // and offer Take latest against the older head. + if (opening != null && !diffDrawer?.ownsOpening(opening)) return deployedValue = replaceFalseWithUndefined({ ...flow, edited_at: undefined, @@ -427,20 +495,59 @@ workspace_id: undefined }) deployedBy = flow.edited_by + deployedVersionShown = flow.version_id + // Names the deployed side of the diff. Without it the reader is shown two panes + // and told nothing about what the left one is. + deployedLabel = `Deployed${flow.version_id != null ? ` ${flow.version_id}` : ''}${flow.edited_by ? ` by ${flow.edited_by}` : ''} · latest` } - async function saveFlow(deploymentMsg?: string, triggersToDeploy?: Trigger[]): Promise { - if (!triggersToDeploy) { - // Check if there are draft triggers that need confirmation + async function saveFlow(deploymentMsg?: string, toDeploy?: DraftChangesToDeploy): Promise { + if (!toDeploy) { + // Draft triggers and drafts on the agents this flow links: both are unsaved changes the + // deploy would otherwise leave behind, so they are confirmed together. const draftTriggers = triggersState.triggers.filter((trigger) => trigger.draftConfig) - if (draftTriggers.length > 0) { - draftTriggersModalOpen = true - confirmDeploymentCallback = async (triggersToDeploy: Trigger[]) => { - await saveFlow(deploymentMsg, triggersToDeploy) + try { + draftAgents = [ + ...( + await loadLinkedAgentDrafts(linkedAgentPaths(flowStore.val.value), opWorkspace) + ).values() + ] + } catch (err: any) { + // This runs before the try below, and `withAIChangesWarning` invokes its callback without + // awaiting, so a rejection here would be unhandled: the button would do nothing at all, + // with no toast and no `onDeployError`. Report it the way the rest of the save does. + // Deploying anyway is not the fallback — this throws only when an agent's unsaved changes + // cannot be read, which is exactly when the dialog must not claim there are none. + onDeployError?.({ error: err }) + sendUserToast(`The flow could not be saved: ${err?.body ?? err}`, true) + return + } + agentCanWrite = {} + agentRefusal = {} + if (draftAgents.length > 0) { + // One lookup for the whole list: an agent lives in a folder, and the groups and admin + // flag that answer for it are per workspace, so the nav user would answer for the wrong + // membership when a session editor operates on another workspace. + const user = await getUserExt(opWorkspace ?? '').catch(() => undefined) + agentCanWrite = Object.fromEntries( + draftAgents.map((a) => [a.path, agentDraftCanWrite(a, user ?? $userStore ?? undefined)]) + ) + // The path is passed, so a draft that renames the agent is refused here too: a rename is + // the agent editor's to deploy, and this dialog lists the agent under the path the flow + // links. + agentRefusal = Object.fromEntries( + draftAgents.map((a) => [a.path, agentDraftDeployRefusal(a.state, a.path)]) + ) + } + if (draftTriggers.length > 0 || draftAgents.length > 0) { + draftChangesModalOpen = true + confirmDeploymentCallback = async (confirmed: DraftChangesToDeploy) => { + await saveFlow(deploymentMsg, confirmed) } return } } + const triggersToDeploy = toDeploy?.triggers loadingSave = true try { @@ -471,6 +578,11 @@ // loadingSave = false // del // return + // Ahead of the flow itself, as the update branch deploys its triggers: an agent is a + // resource of its own, so the flow should land on top of the agent set it was tested + // against rather than the other way round. + await deployAgentDrafts(toDeploy?.agents ?? []) + // `newFlow` comes from the embedder, and updating a path that has no // deployed flow 404s. Confirm with the server before taking the update // branch so a first deploy still lands. @@ -518,7 +630,7 @@ await deployTriggers( triggersToDeploy, opWorkspace, - !!$userStore?.is_admin || !!$userStore?.is_super_admin, + !!actingUser?.is_admin || !!actingUser?.is_super_admin, usedTriggerKinds, $pathStore, true @@ -529,7 +641,7 @@ await deployTriggers( triggersToDeploy, opWorkspace, - !!$userStore?.is_admin || !!$userStore?.is_super_admin, + !!actingUser?.is_admin || !!actingUser?.is_super_admin, usedTriggerKinds, initialPath ) @@ -648,7 +760,7 @@ const resolved = linkedAgentToolsForScope(to) for (const [moduleId, agentPath] of linkedAgentEntries(linkedAgentRefs)) { if (resolved[moduleId] === undefined) { - publishLinkedAgentTools(agentPath, ws, to, moduleId) + publishLinkedAgentTools(agentPath, ws, to, moduleId, true) } } } @@ -716,7 +828,7 @@ // writing overrides against — the tool ids of the agent that was just replaced. claimLinkedToolsFetch(scope, moduleId) clearLinkedAgentTools(scope, moduleId) - publishLinkedAgentTools(agentPath, ws, scope, moduleId) + publishLinkedAgentTools(agentPath, ws, scope, moduleId, true) } publishedAgentByModule = next }) @@ -731,6 +843,12 @@ return entries } + // The storage path this editor is bound to, narrowing whatever an outer mount + // published (a session tab): the full-page editor holds its own, a drawer mount + // holds none. + const outerStoragePath = getEditorStoragePath() + setEditorStoragePath(() => liveEditorDraftStoragePath ?? outerStoragePath?.()) + // "Open in AI session" target: the URL draft path the editor loads/saves by // (which for a new flow differs from the live-edited friendly `$pathStore`), // falling back to `$pathStore` in drawer mounts that carry no storage path. @@ -1020,21 +1138,117 @@ } } - async function openDiffDrawer() { + /** Deployed versions for the diff picker, newest first. Best-effort: losing the + * list costs the picker, not the diff. */ + /** Throws: the drawer says so and lets the reader ask for the same page again. */ + async function fetchVersionPage(page: number) { + const path = userDraftPath || initialPath + if (!opWorkspace || !path) return undefined + { + const history = await FlowService.getFlowHistory({ + workspace: opWorkspace, + path, + page, + perPage: VERSION_PAGE_SIZE + }) + // Head is the version the payload beside this list came from, not whatever the + // history now leads with: a deploy landing between the two fetches would + // otherwise label the shown (older) value as the latest. + const head = deployedVersionShown ?? history[0]?.id + // No ordinal: the list arrives a page at a time, so a number counted within one + // would rename versions as more load. + return history.map((h) => { + const detail = [ + h.created_by, + h.created_at ? new Date(h.created_at).toLocaleString() : undefined, + h.deployment_msg + ].filter(Boolean) + const isHead = h.id === head + return { + id: String(h.id), + label: `${h.id}${isHead ? ' · latest' : ''}`, + subtitle: detail.length ? detail.join(' · ') : undefined, + isHead + } + }) + } + } + + /** The first page, best-effort: losing it costs the picker, not the diff. */ + async function deployedVersionOptions() { + try { + return await fetchVersionPage(1) + } catch { + return undefined + } + } + + /** Hands the drawer the next page each time the reader asks for one. The page number + * belongs to this item's history, so it lives here — and only moves once a page has + * actually arrived, or a failed request would skip it. */ + function moreVersionsLoader() { + let loaded = 1 + return async () => { + const page = await fetchVersionPage(loaded + 1) + loaded += 1 + return page + } + } + + /** The opening this editor claimed last. A path change remounts this editor while the + * drawer stays mounted, so its teardown hands that opening back rather than leaving + * the drawer on the item the user left. */ + let lastOpening: number | undefined = undefined + onDestroy(() => { + if (lastOpening != null) diffDrawer?.abandonOpening(lastOpening) + }) + + export async function openDiffDrawer() { if (!savedFlow) return - await syncWithDeployed() + // The fetches below are awaited, so a reopen (or a path change, which remounts + // this editor but not the drawer) while they run must not have the older one + // land last. The drawer counts the openings for that reason. + const opening = diffDrawer?.beginOpening() + lastOpening = opening + if (opening == null) return + await syncWithDeployed(opening) const currentDraftTriggers = structuredClone(triggersState.getDraftTriggersSnapshot()) - diffDrawer?.openDrawer() + // Blanking the drawer belongs to the opening that will fill it. + if (!diffDrawer?.ownsOpening(opening)) return + diffDrawer.openDrawer(opening) const currentFlow = flowStore.val - diffDrawer?.setDiff({ - mode: 'normal', - deployed: deployedValue ?? savedFlow, - current: { - ...currentFlow, - path: $pathStore, - draft_triggers: currentDraftTriggers - } - }) + const versions = await deployedVersionOptions() + if (!diffDrawer?.ownsOpening(opening)) return + diffDrawer.setDiff( + { + mode: 'normal', + deployed: deployedValue ?? savedFlow, + deployedLabel, + versions, + loadMoreVersions: moreVersionsLoader(), + onTakeLatest, + draftBase: draftBaseVersion, + deployedHead: deployedVersionShown != null ? String(deployedVersionShown) : undefined, + loadVersion: async (id) => { + const v = await FlowService.getFlowVersion({ + workspace: opWorkspace!, + version: Number(id) + }) + return replaceFalseWithUndefined({ + ...v, + edited_at: undefined, + edited_by: undefined, + workspace_id: undefined + }) + }, + current: { + ...currentFlow, + path: $pathStore, + draft_triggers: currentDraftTriggers + } + }, + opening + ) } let flowCopilotContext: FlowCopilotContext = $state({ @@ -1053,21 +1267,11 @@ setContext('FlowCopilotContext', flowCopilotContext) let renderCount = $state(0) - let flowTutorials: FlowTutorials | undefined = $state(undefined) let jsonViewerDrawer: Drawer | undefined = $state(undefined) let yamlEditorDrawer: Drawer | undefined = $state(undefined) let flowHistory: FlowHistory | undefined = $state(undefined) - export function triggerTutorial() { - const urlParams = new URLSearchParams(window.location.search) - const tutorial = urlParams.get('tutorial') - - if (tutorial) { - flowTutorials?.runTutorialById(tutorial) - } - } - let baseMenuItems: Item[] = $state([]) const mod = isMac() ? '⌘' : 'Ctrl+' @@ -1091,56 +1295,6 @@ disabled: $history.index === $history.history.length - 1, shortcut: `${mod}⇧Z` }, - { - displayName: 'Tutorials', - icon: BookOpen, - separatorTop: true, - extra: (() => { - const remaining = [ - getTutorialIndex('flow-live-tutorial'), - getTutorialIndex('troubleshoot-flow') - ].filter((i) => $tutorialsToDo.includes(i)).length - return remaining > 0 - ? createRawSnippet(() => ({ - render: () => - `${remaining}` - })) - : undefined - })(), - submenuItems: [ - { - displayName: 'Build a flow', - action: () => flowTutorials?.runTutorialById('flow-live-tutorial'), - icon: $tutorialsToDo.includes(getTutorialIndex('flow-live-tutorial')) - ? Circle - : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('flow-live-tutorial')) - ? undefined - : 'green' - }, - { - displayName: 'Fix a broken flow', - action: () => flowTutorials?.runTutorialById('troubleshoot-flow'), - icon: $tutorialsToDo.includes(getTutorialIndex('troubleshoot-flow')) - ? Circle - : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('troubleshoot-flow')) - ? undefined - : 'green' - }, - { - displayName: 'Reset tutorials', - action: () => resetAllTodos(), - icon: RefreshCw, - separatorTop: true - }, - { - displayName: 'Skip tutorials', - action: () => skipAllTodos(), - icon: CheckCheck - } - ] - }, { displayName: 'Test flow & record', icon: Disc, @@ -1339,24 +1493,28 @@ {confirmCallback} bind:open {diffDrawer} + claimOpening={() => (lastOpening = diffDrawer?.beginOpening())} bind:deployedValue currentValue={flowStore.val} /> - t.draftConfig)} + {draftAgents} + {agentCanWrite} + {agentRefusal} isFlow={true} on:canceled={() => { - draftTriggersModalOpen = false + draftChangesModalOpen = false }} - on:confirmed={handleDraftTriggersConfirmed} + on:confirmed={handleDraftChangesConfirmed} /> {#key renderCount} - {#if !$userStore?.operator} + {#if !actingUser?.operator} {#if $pathStore} {/if} @@ -1407,14 +1565,7 @@ {#if $enterpriseLicense && !newFlow && !inSessionPane} {/if} -
- - {#if $tutorialsToDo.includes(getTutorialIndex('flow-live-tutorial')) || $tutorialsToDo.includes(getTutorialIndex('troubleshoot-flow'))} - - {/if} -
+ {#if diffEnabled && !diffInMenu} +
-
- {#if stepDetail.id && stepDetail.id != 'failure' && stepDetail.id != 'preprocessor'} - - {stepDetail.id} - - {/if} - - {#if stepDetail.summary} - {stepDetail.summary} - {:else if stepDetail.value.type == 'identity'} - Identity - {:else if stepDetail.value.type == 'forloopflow'} - For loop {#if stepDetail.value.parallel}(parallel){/if} - {#if stepDetail.value.skip_failures}(skip failures){/if} - {#if stepDetail.value.squash}(squash){/if} - {:else if stepDetail.value.type == 'branchall'} - Run all branches {#if stepDetail.value.parallel}(parallel){/if} - {:else if stepDetail.value.type == 'branchone'} - Run one branch - {:else if stepDetail.value.type == 'flow'} - Inner flow - {:else if stepDetail.value.type == 'whileloopflow'} - While loop {#if stepDetail.value.skip_failures}(skip failures){/if} - {#if stepDetail.value.squash}(squash){/if} - {:else if stepDetail.id === 'failure'} - Error handler - {:else if stepDetail.id === 'preprocessor'} - Preprocessor - {:else if stepDetail.value.type == 'rawscript'} - Inline {stepDetail.value.language} script - {:else if stepDetail.value.type == 'script'} - Workspace script - {:else if stepDetail.value.type == 'aiagent'} - AI Agent - {/if} - -
{#if stepDetail.value.type == 'script'}
+ import type { FlowModule } from '$lib/gen' + import { Badge, Button } from './common' + import { ArrowLeft } from 'lucide-svelte' + + interface Props { + /** A module, or the graph's pseudo-nodes by id (`Input`, `Result`). */ + stepDetail: FlowModule | string + /** Given, the row starts with a back control; the caller decides where back leads. */ + onBack?: () => void + } + + let { stepDetail, onBack = undefined }: Props = $props() + + const module = $derived(typeof stepDetail === 'string' ? undefined : stepDetail) + // The error handler and the preprocessor are named by their role, not by an id badge. + const showId = $derived( + module?.id !== undefined && module.id !== 'failure' && module.id !== 'preprocessor' + ) + + const title = $derived.by((): string => { + if (typeof stepDetail === 'string') { + if (stepDetail === 'Input') return 'Flow inputs' + if (stepDetail === 'Result') return 'Result' + return stepDetail + } + if (stepDetail.summary) return stepDetail.summary + if (stepDetail.id === 'failure') return 'Error handler' + if (stepDetail.id === 'preprocessor') return 'Preprocessor' + const v = stepDetail.value + switch (v?.type) { + case 'identity': + return 'Identity' + case 'forloopflow': + return ( + 'For loop' + + (v.parallel ? ' (parallel)' : '') + + (v.skip_failures ? ' (skip failures)' : '') + + (v.squash ? ' (squash)' : '') + ) + case 'whileloopflow': + return ( + 'While loop' + (v.skip_failures ? ' (skip failures)' : '') + (v.squash ? ' (squash)' : '') + ) + case 'branchall': + return 'Run all branches' + (v.parallel ? ' (parallel)' : '') + case 'branchone': + return 'Run one branch' + case 'flow': + return 'Inner flow' + case 'rawscript': + return `Inline ${v.language} script` + case 'script': + return 'Workspace script' + case 'aiagent': + return 'AI Agent' + default: + return stepDetail.id + } + }) + + + +
+ {#if onBack} +
diff --git a/frontend/src/lib/components/FlowHistoryJobPicker.svelte b/frontend/src/lib/components/FlowHistoryJobPicker.svelte index 26fa41f1af..dfb4927af1 100644 --- a/frontend/src/lib/components/FlowHistoryJobPicker.svelte +++ b/frontend/src/lib/components/FlowHistoryJobPicker.svelte @@ -3,9 +3,11 @@ import { createEventDispatcher, untrack } from 'svelte' import PopoverV2 from '$lib/components/meltComponents/Popover.svelte' import HistoricInputs from './HistoricInputs.svelte' - import { workspaceStore } from '$lib/stores' import { JobService } from '$lib/gen' import { Button } from './common' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { path: string @@ -27,7 +29,7 @@ async function loadInitial() { loading = true let jobs = await JobService.listJobs({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, scriptPathExact: path, jobKinds: ['flow', 'flowpreview'].join(','), perPage: 1 @@ -43,7 +45,7 @@ } $effect(() => { - if ($workspaceStore && !newFlow) { + if ($operatingWorkspace && !newFlow) { untrack(() => loadInitial()) } }) diff --git a/frontend/src/lib/components/FlowJobResult.svelte b/frontend/src/lib/components/FlowJobResult.svelte index eea95a6ecf..66e30aefec 100644 --- a/frontend/src/lib/components/FlowJobResult.svelte +++ b/frontend/src/lib/components/FlowJobResult.svelte @@ -2,10 +2,7 @@ import { Loader2 } from 'lucide-svelte' import DisplayResult from './DisplayResult.svelte' import LogViewer from './LogViewer.svelte' - import type { CompletedJob, Job } from '$lib/gen' - import AiAgentLogViewer from './AIAgentLogViewer.svelte' import { twMerge } from 'tailwind-merge' - import type { AgentTool } from './flows/agentToolUtils' interface Props { waitingForExecutor?: boolean @@ -16,17 +13,13 @@ loading: boolean filename?: string | undefined jobId?: string | undefined + /** Identifies the run, which `jobId` cannot on the replay page. */ + runKey?: string | undefined tag?: string | undefined workspaceId?: string | undefined refreshLog?: boolean downloadLogs?: boolean tagLabel?: string | undefined - aiAgentStatus?: { - tools: AgentTool[] - agentJob: Partial & Pick & { type: 'CompletedJob' } - storedToolCallJobs?: Record - onToolJobLoaded?: (job: Job, idx: number) => void - } } let { @@ -38,11 +31,11 @@ loading, filename = undefined, jobId = undefined, + runKey = undefined, tag = undefined, workspaceId = undefined, downloadLogs = true, - tagLabel = undefined, - aiAgentStatus = undefined + tagLabel = undefined }: Props = $props() @@ -60,7 +53,15 @@ : 'max-h-80'} overflow-auto rounded-md grow min-h-0 border bg-surface-tertiary p-2" > {#if result !== undefined || result_stream !== undefined} - + {:else if loading} {:else} @@ -70,19 +71,15 @@
Logs - {#if aiAgentStatus} - - {:else} -
- -
- {/if} +
+ +
diff --git a/frontend/src/lib/components/FlowLogViewer.svelte b/frontend/src/lib/components/FlowLogViewer.svelte index 385d028e7e..6a8db18a76 100644 --- a/frontend/src/lib/components/FlowLogViewer.svelte +++ b/frontend/src/lib/components/FlowLogViewer.svelte @@ -11,7 +11,6 @@ Keyboard } from 'lucide-svelte' import { base } from '$lib/base' - import { workspaceStore } from '$lib/stores' import ObjectViewer from './propertyPicker/ObjectViewer.svelte' import LogViewer from './LogViewer.svelte' import FlowLogViewer from './FlowLogViewer.svelte' @@ -26,6 +25,9 @@ import { Tooltip } from './meltComponents' import FlowTimelineBar from './FlowTimelineBar.svelte' import { getActiveReplay } from './recording/replay.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() type RootJobData = Partial @@ -49,7 +51,6 @@ ) => Promise getSelectedIteration: (stepId: string) => number flowSummary?: string - mode?: 'flow' | 'aiagent' currentId?: string | null navigationChain?: NavigationChain select: (id: string) => void @@ -81,7 +82,6 @@ onSelectedIteration, getSelectedIteration, flowSummary, - mode = 'flow', currentId, navigationChain = $bindable(), select, @@ -98,7 +98,7 @@ function getJobLink(jobId: string | undefined): string { if (!jobId) return '' - return `${base}/run/${jobId}?workspace=${workspaceId ?? $workspaceStore}` + return `${base}/run/${jobId}?workspace=${workspaceId ?? $operatingWorkspace}` } function getStatusColor(status: FlowStatusModule['type'] | undefined): string { @@ -127,18 +127,16 @@ function getStepProgress(job: RootJobData | undefined, totalSteps: number): string { if (!job || totalSteps === 0) return '' - const stepWord = mode === 'aiagent' ? 'action' : 'step' - // If flow is completed, show total steps if (job.type === 'CompletedJob') { - return ` (${totalSteps} ${stepWord}${totalSteps === 1 ? '' : 's'})` + return ` (${totalSteps} step${totalSteps === 1 ? '' : 's'})` } // If flow is running, use flow_status.step if available (like JobStatus.svelte) if (job.type === 'QueuedJob') { if (job.flow_status?.step !== undefined) { const currentStep = (job.flow_status.step ?? 0) + 1 - return ` (${stepWord} ${currentStep} of ${totalSteps})` + return ` (step ${currentStep} of ${totalSteps})` } return '' @@ -558,7 +556,7 @@ {@render flowIcon(getFlowStatus(rootJob), flowInfo?.hasErrors)}
- {mode === 'aiagent' ? 'AI Agent' : level == 0 ? 'Flow' : 'Subflow'} + {level == 0 ? 'Flow' : 'Subflow'} {#if flowInfo?.label} : {flowInfo.label} {/if} @@ -703,32 +701,22 @@
- {#if mode === 'aiagent'} - {#if module.summary} - Tool call: {module.summary} - {:else} - Message - {/if} - {:else} - {module.id} - {/if} + {module.id} - {#if mode === 'flow'} - {#if module.value.type === 'forloopflow'} - For loop - {:else if module.value.type === 'whileloopflow'} - While loop - {:else if module.value.type === 'branchall'} - Branch to all - {:else if module.value.type === 'branchone'} - Branch to one - {:else if module.value.type === 'flow'} - Subflow - {:else} - Step - {/if} + {#if module.value.type === 'forloopflow'} + For loop + {:else if module.value.type === 'whileloopflow'} + While loop + {:else if module.value.type === 'branchall'} + Branch to all + {:else if module.value.type === 'branchone'} + Branch to one + {:else if module.value.type === 'flow'} + Subflow + {:else} + Step {/if} - {#if module.summary && mode !== 'aiagent'} + {#if module.summary} : {module.summary} {/if} {#if hasEmptySubflowValue} diff --git a/frontend/src/lib/components/FlowLogViewerWrapper.svelte b/frontend/src/lib/components/FlowLogViewerWrapper.svelte index 648dbc9f06..12abe9bfb2 100644 --- a/frontend/src/lib/components/FlowLogViewerWrapper.svelte +++ b/frontend/src/lib/components/FlowLogViewerWrapper.svelte @@ -20,7 +20,6 @@ | { id: string; index: number; manuallySet: true; moduleId: string } | { manuallySet: false; moduleId: string } ) => Promise - mode?: 'flow' | 'aiagent' } let { @@ -29,8 +28,7 @@ localDurationStatuses, workspaceId, render, - onSelectedIteration, - mode = 'flow' + onSelectedIteration }: Props = $props() // State for tracking expanded rows - using Record to allow explicit control @@ -180,7 +178,6 @@ {render} {getSelectedIteration} flowId="root" - {mode} {currentId} bind:navigationChain {select} diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index 1378e20f04..f136cbfe39 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -1,15 +1,18 @@ - - \ No newline at end of file diff --git a/frontend/src/lib/components/FlowViewer.svelte b/frontend/src/lib/components/FlowViewer.svelte index 1e321584e3..c68d62ad42 100644 --- a/frontend/src/lib/components/FlowViewer.svelte +++ b/frontend/src/lib/components/FlowViewer.svelte @@ -5,13 +5,16 @@ import SchemaViewer from './SchemaViewer.svelte' import FlowGraphViewer from './FlowGraphViewer.svelte' import { Loader2 } from 'lucide-svelte' - import { orderedYamlStringify, cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' - import { workspaceStore } from '$lib/stores' + import { cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { watch } from 'runed' import HighlightTheme from './HighlightTheme.svelte' import FlowViewerInner from './FlowViewerInner.svelte' import FlowInputViewer from './FlowInputViewer.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface PreviousFlow { summary: string @@ -85,7 +88,7 @@ return } previousFlow = await FlowService.getFlowVersion({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, version }) previousFlowCache[version] = previousFlow diff --git a/frontend/src/lib/components/FolderEditor.svelte b/frontend/src/lib/components/FolderEditor.svelte index dede17b953..7af2066fa6 100644 --- a/frontend/src/lib/components/FolderEditor.svelte +++ b/frontend/src/lib/components/FolderEditor.svelte @@ -1,24 +1,29 @@ - - { - newGroup?.closeDrawer() - groupCreated = undefined - }} - > - {#if !groupCreated} -
- - -
- {:else} - - {/if} -
-
- - - - - - +
- + {/if} + +
{#if can_write} - + (pendingLabel = v)} + /> {:else}
- {#each labels ?? [] as label (label)} + {#each draft.labels as label (label)} {label} {:else} No labels @@ -319,257 +627,256 @@
-
- - - + Name + Kind + Role + Actions - {/snippet} - {#snippet body()} - - {#each perms ?? [] as { owner_name, role }} - - + {#each draft.perms as perm, idx (perm.owner_name)} + + + {ownerNameOf(perm.owner_name)} + + {ownerKindOf(perm.owner_name) === 'group' ? 'Group' : 'User'} + + {#if can_write && !restricted} +
+ { + draft.perms[idx].role = e.detail + }} + > + {#snippet children({ item })} + - + - - {/snippet} - -
- {:else} - {role} - {/if} -
- {/each} - - {/snippet} - - - - {:else if folderNotFound === undefined} + + {:else}
{#each new Array(6) as _} @@ -580,7 +887,10 @@ {#if canEditDefaults} - + defaultRulesOpen || defaultRulesInvalid, (v) => (defaultRulesOpen = v)} + text="Default permissioned as (advanced, prod only)" + >
This setting is mostly relevant on production workspaces where you want @@ -592,30 +902,33 @@ never rewritten. - {#if defaultPermissionedAs.length > 0} - - {#snippet headerRow()} + {#if draft.defaultPermissionedAs.length > 0} + +
- - - + + path_glob Glob relative to f/{name}/ + + Permissioned as + Actions - {/snippet} - {#snippet body()} - - {#each defaultPermissionedAs as rule, idx (idx)} - {@const kind = ruleKind(rule.permissioned_as)} - {@const itemsForKind = kind === 'user' ? usernames : groups} - - - + {#each draft.defaultPermissionedAs as rule, idx (idx)} + {@const kind = ownerKindOf(rule.permissioned_as)} + {@const itemsForKind = kind === 'user' ? usernames : groups} + + + + + +
+
setRulePermissionedAs(idx, e.detail, '')} @@ -625,49 +938,51 @@ {/snippet} -
- - {/each} - - {/snippet} - + ({ + label: p.path_with_namespace, + value: p.path_with_namespace + }))} + bind:value={selectedProject} + clearable={false} + /> + + {#if applyError} + {applyError} + {/if} +
+ +
+ {/if} + + + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/GitRepoPopoverPicker.svelte b/frontend/src/lib/components/GitRepoPopoverPicker.svelte index f8050a90a2..a8eeeb3fe9 100644 --- a/frontend/src/lib/components/GitRepoPopoverPicker.svelte +++ b/frontend/src/lib/components/GitRepoPopoverPicker.svelte @@ -1,9 +1,11 @@
@@ -124,207 +377,218 @@ permission, deployed items will be reassigned to the deploying user. {/if} - + {/if} + + -
- - - - - {/snippet} - {#snippet body()} - - {#each members ?? [] as { member_name, role }} - - + Name + Role + Actions + + + + {#each draft.members as member, idx (member.member_name)} + + + {member.member_name} + + + {#if can_write && !restricted}
{ - const role = e.detail - // const wasInGroup = (group?.members ?? []).includes(group) - // const inAcl = ( - // group?.extra_perms ? Object.keys(group?.extra_perms) : [] - // ).includes(group) - if (role == 'member') { - await GroupService.addUserToGroup({ - workspace: $workspaceStore ?? '', - name, - requestBody: { - username: member_name - } - }) - await GranularAclService.removeGranularAcls({ - workspace: $workspaceStore ?? '', - path: name, - kind: 'group_', - requestBody: { - owner: 'u/' + member_name - } - }) - } else if (role == 'manager') { - await GroupService.removeUserToGroup({ - workspace: $workspaceStore ?? '', - name, - requestBody: { - username: member_name - } - }) - await GranularAclService.addGranularAcls({ - workspace: $workspaceStore ?? '', - path: name, - kind: 'group_', - requestBody: { - owner: 'u/' + member_name, - write: true - } - }) - } else if (role == 'admin') { - await GroupService.addUserToGroup({ - workspace: $workspaceStore ?? '', - name, - requestBody: { - username: member_name - } - }) - await GranularAclService.addGranularAcls({ - workspace: $workspaceStore ?? '', - path: name, - kind: 'group_', - requestBody: { - owner: 'u/' + member_name, - write: true - } - }) - } - loadGroup() + disabled={isFixedCreatorRow(member.member_name)} + selected={member.role} + on:selected={(e) => { + draft.members[idx].role = e.detail }} > {#snippet children({ item })} - {#if role === 'manager'} + + {#if member.role === 'manager'} {/if} {/snippet}
{:else} - {role} - {/if} -
- {/each} + {member.role} + {/if} + + +
+ {#if can_write && !isFixedCreatorRow(member.member_name)} +
+
+ + {/each}
- {/snippet} - - - {#if instance_group?.emails} -

Members from the instance group

- - {#snippet headerRow()} -
- - - {/snippet} - {#snippet body()} - - {#each instance_group?.emails ?? [] as email} - - {/each} - - {/snippet} - + + {:else} +
+ {#each new Array(6) as _} + + {/each} +
{/if} - {:else} -
- {#each new Array(6) as _} - - {/each} -
- {/if} + + {#if instance_group?.emails} +
+ Email + + + + {#each instance_group?.emails ?? [] as email} + + {email} + + {/each} + + + + {/if} + {#if reloadHistory > 0} {#key reloadHistory} + import { Button, Drawer, DrawerContent } from './common' + import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' + import GroupEditor from './GroupEditor.svelte' + import { Save } from 'lucide-svelte' + import { sendUserToast } from '$lib/toast' + + let { + offset = 0, + disableChatOffset = false, + onSaved = undefined + }: { + offset?: number + disableChatOffset?: boolean + onSaved?: (name: string, created: boolean) => void | Promise + } = $props() + + let drawer: Drawer | undefined = $state() + let mode: 'edit' | 'new' = $state('edit') + let name: string = $state('') + let canSave = $state(false) + let unsaved = $state(false) + // A `new` drawer whose group has been created but whose member calls then failed stays + // open on the edit path. Calling it Create there would offer to create what exists. + let exists = $state(false) + let saving = $state(false) + let confirmDiscardOpen = $state(false) + let discarding = $state(false) + let editor: { save: () => Promise<{ name: string; created: boolean } | undefined> } | undefined = + $state() + // Bumped per open so the editor reloads its draft from the group it is now pointed at. + // Keying on `name` instead would remount on every keystroke of the name field in `new` mode. + let instance = $state(0) + + function open(nextMode: 'edit' | 'new', groupName: string): void { + mode = nextMode + name = groupName + discarding = false + confirmDiscardOpen = false + exists = nextMode === 'edit' + // The remounted editor reports these on its first effect, which is a tick away. Until + // then the header would carry the last group's answers. + canSave = false + unsaved = false + instance++ + drawer?.openDrawer() + } + + export function initEdit(groupName: string): void { + open('edit', groupName) + } + + export function initNew(initialName: string = ''): void { + open('new', initialName) + } + + /** The editor keeps its draft in memory only, so closing throws it away. */ + function requestClose() { + // A save is already writing. `unsaved` only clears once it reloads, so closing here + // would offer to discard changes the in-flight requests are busy persisting — and + // confirming would close on that lie. Saving is the shorter wait; ignore the close. + if (saving) return + if (discarding || !unsaved) { + drawer?.closeDrawer() + return + } + confirmDiscardOpen = true + } + + async function save() { + saving = true + try { + const saved = await editor?.save() + if (saved) { + // Callers reload a list here. Called from inside the chain, not before it, so a + // synchronous throw is caught too — thrown out of `save()` it would skip the + // close below and strand the drawer open on a group that did save. + void Promise.resolve() + .then(() => onSaved?.(saved.name, saved.created)) + .catch((e) => sendUserToast(e?.body ?? String(e), true)) + // The editor reloads its baseline after saving, but that lands a tick later; + // close on our own authority rather than racing it. + discarding = true + // Belt and braces with the `saving` guard on the close paths: nothing that + // asked to discard may outlive a save that then succeeded. + confirmDiscardOpen = false + drawer?.closeDrawer() + } + } finally { + saving = false + } + } + + + { + // Escape and click-away close the drawer before asking. Reopening in the same tick is + // how the flow's script editor drawer handles this too: the close transition has not + // started, so nothing flickers. + if (saving) { + drawer?.openDrawer() + return + } + if (!discarding && unsaved) { + drawer?.openDrawer() + confirmDiscardOpen = true + } + }} +> + + +
+ {#key instance} + (canSave = v)} + onUnsavedChange={(v) => (unsaved = v)} + onExistsChange={(v) => (exists = v)} + /> + {/key} +
+ {#snippet actions()} + + {/snippet} +
+
+ + + (confirmDiscardOpen = false)} + onConfirmed={() => { + confirmDiscardOpen = false + discarding = true + drawer?.closeDrawer() + }} +> + Are you sure you want to discard the changes you have made to this group? + diff --git a/frontend/src/lib/components/HistoricInputs.svelte b/frontend/src/lib/components/HistoricInputs.svelte index 85f2603005..48d1290356 100644 --- a/frontend/src/lib/components/HistoricInputs.svelte +++ b/frontend/src/lib/components/HistoricInputs.svelte @@ -7,7 +7,9 @@ import { DataTable } from '$lib/components/table' import HistoricList from './HistoricList.svelte' import { Loader2 } from 'lucide-svelte' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { runnableId?: string | undefined @@ -32,7 +34,7 @@ workspace = undefined }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) let historicList: HistoricList | undefined = $state(undefined) const dispatch = createEventDispatcher() diff --git a/frontend/src/lib/components/HistoricList.svelte b/frontend/src/lib/components/HistoricList.svelte index bb34daee77..89da866fc2 100644 --- a/frontend/src/lib/components/HistoricList.svelte +++ b/frontend/src/lib/components/HistoricList.svelte @@ -1,9 +1,11 @@ diff --git a/frontend/src/lib/components/ImportProjectCard.svelte b/frontend/src/lib/components/ImportProjectCard.svelte index 26bb3f7b18..92b82f3163 100644 --- a/frontend/src/lib/components/ImportProjectCard.svelte +++ b/frontend/src/lib/components/ImportProjectCard.svelte @@ -23,9 +23,13 @@ project: ImportProjectSummary /** Where the project is coming from, shown next to the author. */ hubHost?: string + /** The project's prose, when the caller has it. Falls back to the one-line summary. */ + description?: string + /** Off where what the import will create is already spelled out below the card. */ + showCounts?: boolean } - let { project, hubHost = 'hub.windmill.dev' }: Props = $props() + let { project, hubHost = 'hub.windmill.dev', description, showCounts = true }: Props = $props() // Protocol-relative on purpose: the same hub is https in production and plain // http when it's a local dev instance, and this way the link follows whichever @@ -83,7 +87,7 @@ class="shrink-0 text-tertiary opacity-0 transition group-hover:opacity-100" />
-

{project.summary}

+

{description || project.summary}

by {project.author} · {project.slug} @@ -91,9 +95,11 @@ -

- -
+ {#if showCounts} +
+ +
+ {/if} diff --git a/frontend/src/lib/components/ImportProjectStep.svelte b/frontend/src/lib/components/ImportProjectStep.svelte index 04c169d41c..f609aff6a3 100644 --- a/frontend/src/lib/components/ImportProjectStep.svelte +++ b/frontend/src/lib/components/ImportProjectStep.svelte @@ -27,6 +27,27 @@ /** From the hub, for the counts — the export is only fetched during the run. */ project?: ImportProjectSummary onFolderChange: (folder: string) => void + /** + * Whether to ask which folder the project lands in. Off where the destination was + * not chosen either — importing into the workspace you are already in is one + * decision, and `f/` is the answer nobody needs to be asked for. + */ + chooseFolder?: boolean + /** + * Whether to spell out what import does to resources and triggers. It is about landing + * on top of what a workspace already holds — a resource it will not overwrite, a + * trigger it re-creates disabled — so a destination with nothing in it has nothing to + * warn about, and the setup step that follows is where the values get filled in. + */ + showNotes?: boolean + /** + * Fill the height given rather than hugging the content, with the actions pinned to the + * bottom. For a surface of a fixed size — a paged dialog, whose height is the taller + * page — where content-height buttons would float mid-panel. `sticky` as well as + * `mt-auto`: a page taller than the box scrolls, and a row that only sat at the end of + * the content would scroll out of reach with it. + */ + fillHeight?: boolean onFinish: () => void /** True once the run reveals data tables the destination has yet to configure. */ setupPending?: boolean @@ -51,6 +72,9 @@ onFolderChange, onFinish, onBack, + chooseFolder = true, + showNotes = true, + fillHeight = false, setupPending = false, setupUndecided = false, onExecution, @@ -312,12 +336,12 @@ } -
+
- {#if existingWorkspace} + {#if existingWorkspace && chooseFolder}
- - Resources are imported as empty stubs — set their values after import; one whose path is - already in the workspace is left exactly as it is and reported as already there, so a value - you have since filled in is never overwritten. Trigger kinds are - recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at creation - and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP and Azure - triggers all require Enterprise. Triggers that reference a resource depend on stubs imported - empty, so fill in the resource value before re-enabling the trigger. - + {#if showNotes} + + + Resources are imported as empty stubs — set their values after import; one whose path is + already in the workspace is left exactly as it is and reported as already there, so a value + you have since filled in is never overwritten. Trigger kinds are recreated disabled, except + GCP and Azure triggers, which manage cloud subscriptions at creation and must be re-created + manually after filling their resource. Kafka, NATS, SQS, GCP and Azure triggers all require + Enterprise. Triggers that reference a resource depend on stubs imported empty, so fill in the + resource value before re-enabling the trigger. + + {/if} -
+
{#if !execution?.done} diff --git a/frontend/src/lib/components/ImportSetupStep.svelte b/frontend/src/lib/components/ImportSetupStep.svelte index 86670b0817..f85ebf7395 100644 --- a/frontend/src/lib/components/ImportSetupStep.svelte +++ b/frontend/src/lib/components/ImportSetupStep.svelte @@ -14,16 +14,25 @@ import IconedResourceType from '$lib/components/IconedResourceType.svelte' import ImportSetupRow from '$lib/components/ImportSetupRow.svelte' import AppConnectDrawer from '$lib/components/AppConnectDrawer.svelte' + import Modal2 from '$lib/components/common/modal/Modal2.svelte' + import Select from '$lib/components/select/Select.svelte' + import { applyRetarget, seesWholeWorkspace } from '$lib/importWizard/retargetDeployed' import { OauthService } from '$lib/gen' import { registryCcCapableFor } from '$lib/components/oauthRegistry' - import { resourceTypeDisplayName } from '$lib/components/resourceTypeDisplay' + import { + addResourceTypeDisplayName, + resourceTypeDisplayName + } from '$lib/components/resourceTypeDisplay' + import { loadResourceTypeDisplayName } from '$lib/components/displayNameLoaders' import { applyOneMigration } from '$lib/components/workspaceSettings/projectInstall' import { probeMigrationsApplied } from '$lib/importWizard/probe' import { + projectReferencesResource, retargetProjectExport, type ProjectExport, type ProjectMigration } from '$lib/components/workspaceSettings/projectBundle' + import { superadmin, userStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { escapeHtml } from '$lib/utils' @@ -43,12 +52,31 @@ * own slug and `installProject` retargets them, so reading the raw paths here would * look for stubs that are not where they landed. */ folder?: string - onSkip: () => void - onFinish: () => void + /** Left with `outstanding` rows still unfilled, which the caller may want to count. */ + onSkip: (outstanding: number) => void + /** Off where the surface already names the step, e.g. a dialog whose title is it. */ + showHeading?: boolean + /** Fill the height given, actions pinned to the bottom. See ImportProjectStep. */ + fillHeight?: boolean + /** + * Finished. `checked` is false where the export could not be read: the step then has no + * idea what is outstanding, so it offers Finish rather than blocking — and a caller + * counting outcomes must not read that as a step that came out clean. + */ + onFinish: (checked: boolean) => void onBack?: () => void } - let { workspace, slug, folder, onSkip, onFinish, onBack }: Props = $props() + let { + workspace, + slug, + folder, + onSkip, + onFinish, + onBack, + showHeading = true, + fillHeight = false + }: Props = $props() type Row = { name: string @@ -82,6 +110,17 @@ * is absent, and removing the row reports "all set" over a credential nobody filled. */ unreadable?: boolean + /** + * The workspace resource this row was pointed at. The project's items reference it + * directly now, so this is what the row has to say instead of the path it used to name. + */ + reusedFrom?: string + /** + * The empty placeholder is still at this row's path, because the retarget could not + * account for every item that might read it. Worth saying: the workspace has a resource + * on it that looks unfinished and is not. + */ + stubKept?: boolean } let loading = $state(true) @@ -89,9 +128,38 @@ let rows = $state([]) let blanks = $state([]) let projectResources: { path: string; resource_type: string }[] = [] + /** + * The subset of `projectResources` the checklist asks about: the ones something in the + * project actually points at. The rest are created and left alone — see + * `projectReferencesResource`. Kept apart from `projectResources` because the full list + * is still what a stub may not be replaced by. + */ + let askableResources: { path: string; resource_type: string }[] = [] let working = $state(false) let resourceEditor: ResourceEditorDrawer | undefined = $state(undefined) + /** The folder the import wrote into, which is where every rewritable referrer lives. */ + const targetFolder = $derived(folder?.trim() || slug) + + /** + * Resources the workspace already has, by resource type — what a stub can be replaced + * by. Empty for a workspace this import created, which is why the choice is offered + * rather than imposed: with nothing to choose from the button goes straight to the + * editor, exactly as it did before. + */ + let candidates = $state>({}) + /** + * How many candidates are worth reading back to find the unfilled ones. Past this a + * workspace holds too many resources of these types to be the case worth filtering — + * one project's stub offered as another's credential — and they are all offered rather + * than costing a request each. + */ + const CANDIDATE_READ_CAP = 40 + /** The credential row whose choice dialog is open. */ + let choosing = $state(undefined) + let chosenPath = $state(undefined) + let reusing = $state(false) + const pendingTables = $derived(rows.filter((r) => r.status !== 'done')) // Split because the two say different things to the user: one data table was never // created, the other exists and could not be read. Telling someone to set up what they @@ -133,6 +201,14 @@ * first half and Connect disappears on the eight such providers, where it would work. */ const canConnectType = (rt: string) => instanceConnects.has(rt) || registryCcCapableFor(rt) + + // A row blocked by a resource of another type names that type, whose row nothing here reads. + $effect(() => { + for (const b of blanks) { + if (b.occupiedBy) void loadResourceTypeDisplayName(workspace, b.occupiedBy) + } + }) + let appConnect: AppConnectDrawer | undefined = $state(undefined) const customInstanceDbs = resource([() => workspace], SettingService.listCustomInstanceDbs) @@ -248,7 +324,7 @@ // Retargeted the same way the import was, so these are where the stubs actually // landed. `retargetProjectExport` is a no-op when the folder is the slug, which is // every new-workspace import. - const target = folder?.trim() || slug + const target = targetFolder const retargeted = retargetProjectExport(exportData, exportData.project?.slug ?? slug, target) // Contained for the same reason the import contains: a crafted export can name a // path outside the folder, and offering that for editing would reach a resource @@ -256,6 +332,20 @@ projectResources = (retargeted.resources ?? []) .map((r) => ({ path: String(r.path), resource_type: String((r as any).resource_type) })) .filter((r) => r.path.startsWith(`f/${target}/`)) + // Asked against the export as published, not the retargeted copy: a path the project + // spells out in code is not rewritten by the retarget, so only the raw export has + // its references and its resource paths agreeing. `resourceCount` asks the same + // question the same way, and the step and the stepper have to give one answer. + // Paired by position, not by reconstructing the retargeted path: `retargetProjectExport` + // maps `resources` in order, and an external path the bundle pulled in lands at + // `f//` with a `_2` suffix on collision, which no slicing recovers. + const askable = new Set( + (retargeted.resources ?? []) + .map((r, i) => [String(r.path), (exportData.resources ?? [])[i]] as const) + .filter(([, raw]) => raw && projectReferencesResource(exportData, String(raw.path))) + .map(([path]) => path) + ) + askableResources = projectResources.filter((r) => askable.has(r.path)) await refreshBlanks() } catch (e: any) { loadError = e?.body ?? e?.message ?? String(e) @@ -318,8 +408,9 @@ // row is kept instead; it just cannot name which fields are short. let requirementsUnknown = false try { - const schema = (await ResourceService.getResourceType({ workspace, path: r.resource_type })) - ?.schema as { required?: string[] } | undefined + const rt = await ResourceService.getResourceType({ workspace, path: r.resource_type }) + addResourceTypeDisplayName(rt) + const schema = rt?.schema as { required?: string[] } | undefined required = schema?.required ?? [] } catch { requirementsUnknown = true @@ -347,13 +438,19 @@ * it only moves a row from outstanding to done. */ async function refreshBlanks(): Promise { - const fresh = await findBlankResources(projectResources) + const fresh = await findBlankResources(askableResources) const stillBlank = new Map(fresh.map((b) => [b.path, b])) if (blanks.length === 0) { blanks = fresh + await loadCandidates() return } blanks = blanks.map((b) => { + // A row pointed at another resource is settled once its own stub is gone: the + // project's items read the chosen resource and nothing is left at this path. A row + // whose stub was kept is not settled, and re-reading it is how filling that stub in + // finally closes the row. + if (b.reusedFrom && !b.stubKept) return b const f = stillBlank.get(b.path) // Every field the fresh read decides is taken from it, not merged selectively: these // describe what is at the path *now*. Keeping a stale `unreadable` leaves a resource @@ -369,12 +466,15 @@ justSaved: false } } - // Gone from the blank list entirely: it was read, and it is filled. + // Gone from the blank list entirely: it was read, and it is filled. `stubKept` goes + // with it — the placeholder the items this run could not move read is a credential + // now, so there is nothing left to tell anyone to fill in. return { ...b, missing: [], unreadable: undefined, occupiedBy: undefined, + stubKept: undefined, done: true, justSaved: !b.done } @@ -387,6 +487,169 @@ if (row) row.justSaved = false }, 1500) } + await loadCandidates() + } + + /** + * Which existing resources each outstanding row could be replaced by. Re-read on every + * refresh rather than once: a resource created from the editor here is a candidate for + * the rows below it. + * + * The project's own resources are never offered — one of this project's stubs standing + * in for another is a reference to something equally unfilled. + */ + async function loadCandidates(): Promise { + const types = [...new Set(blanks.map((b) => b.resourceType))] + if (types.length === 0) { + candidates = {} + return + } + const own = new Set(projectResources.map((r) => r.path)) + const next: Record = Object.fromEntries(types.map((t) => [t, []])) + try { + // One call for every type at once — `resource_type` takes a comma-separated list — + // and every page of it: `perPage` is what bounds the answer, so without the loop a + // workspace past one page would have the rest of its resources silently hidden. + for (let page = 1; page <= 100; page++) { + const rows = await ResourceService.listResource({ + workspace, + resourceType: types.join(','), + page, + perPage: 100 + }) + for (const r of rows) { + if (own.has(r.path)) continue + next[r.resource_type ?? '']?.push(r.path) + } + if (rows.length < 100) break + } + } catch { + // Offer nothing rather than a partial list: every row then behaves as it did before + // this choice existed, which is a working way to fill a credential. + candidates = {} + return + } + // An unfilled resource is never the answer to "which credential should this use" — + // another project's stub above all, which the path filter above cannot recognise. + const paths = Object.values(next).flat() + if (paths.length <= CANDIDATE_READ_CAP) { + const settled = await Promise.all(paths.map(async (p) => [p, await isUnfilled(p)] as const)) + const unfilled = new Set(settled.filter(([, empty]) => empty).map(([p]) => p)) + for (const t of Object.keys(next)) next[t] = next[t].filter((p) => !unfilled.has(p)) + } + candidates = next + } + + /** + * Whether a resource holds nothing. Same test the checklist uses to call one of the + * project's own resources blank, so a resource this drops is exactly one the wizard + * would have asked someone to fill in. + */ + async function isUnfilled(path: string): Promise { + try { + const found = await ResourceService.getResource({ workspace, path }) + const value = found?.value + if (!value || typeof value !== 'object') return true + return !Object.values(value).some((v) => v !== undefined && v !== null && v !== '') + } catch { + // A read that fails says nothing about the value, and offering it is what this did + // before the check existed. + return false + } + } + + /** + * The row's one action. A workspace that already has a resource of this type gets the + * choice first — reusing what is there is usually the answer, and entering the same + * credentials a second time is the thing worth avoiding. With nothing to choose from + * there is no choice to make, so it goes straight where it always went. + */ + function startFilling(b: Blank): void { + // A kept-stub row has already been pointed at a resource; what is left is the empty + // placeholder the items this run could not move still read. Reusing a second resource + // would move nothing — every rewritable referrer is off the stub — and would relabel + // the row after a retarget that did nothing. + if (b.done || b.stubKept || (candidates[b.resourceType] ?? []).length === 0) { + fillDirectly(b) + return + } + chosenPath = undefined + choosing = b + } + + /** Connect where the instance can, hand-fill otherwise. */ + function fillDirectly(b: Blank): void { + if (canConnectType(b.resourceType)) appConnect?.open(b.resourceType, b.path) + else resourceEditor?.initEdit(b.path) + } + + /** + * The chooser's way out: close it and do what the button did before there was a choice. + * The row is read out of the state first — closing the dialog unmounts the block that + * would otherwise be holding it. + */ + function fillNewInstead(): void { + const b = choosing + choosing = undefined + if (b) fillDirectly(b) + } + + /** + * Point the project at an existing resource: every imported item that referenced the stub + * is rewritten to the chosen path. Nothing is copied. The stub is deleted only when + * `applyRetarget` can account for every item that might read it, and kept otherwise — so + * the toast says how many items moved, and whether the placeholder is still there. + */ + async function reuseChosen(): Promise { + const b = choosing + const target = chosenPath + if (!b || !target) return + reusing = true + working = true + try { + const outcome = await applyRetarget({ + workspace, + folder: targetFolder, + from: b.path, + to: target, + // Asked of this workspace, not of whichever one the user record still describes: + // reloading on this step leaves `$userStore` pointing at the previous workspace. + seesWholeWorkspace: seesWholeWorkspace($userStore, !!$superadmin, workspace) + }) + const moved = `${outcome.rewritten.length} item${outcome.rewritten.length === 1 ? '' : 's'}` + if (outcome.error) { + sendUserToast( + `Could not point the project at ${target}: ${outcome.error}. ${moved} had already been updated, and ${b.path} was kept.`, + true + ) + return + } + choosing = undefined + const row = blanks.find((x) => x.path === b.path) + if (row) { + row.reusedFrom = target + row.stubKept = !outcome.stubDeleted + // Settled only when the stub is gone. A kept stub is empty and is still what + // every item the scan could not move reads, so the row stays outstanding and + // keeps its action: filling it in is the thing left to do. + row.done = outcome.stubDeleted + row.justSaved = outcome.stubDeleted + } + await refreshBlanks() + sendUserToast( + outcome.stubDeleted + ? `The project now uses ${target} — ${moved} updated.` + : `The project now uses ${target} — ${moved} updated. ${b.path} was kept, because some items could not be checked.` + ) + } catch (e: any) { + sendUserToast( + `Could not point the project at ${target}: ${e?.body ?? e?.message ?? String(e)}`, + true + ) + } finally { + reusing = false + working = false + } } $effect(() => { @@ -479,7 +742,7 @@ }) if (!confirmed) return } - onSkip() + onSkip(outstanding) } /** @@ -505,9 +768,11 @@ } -
+
-

Finish setting up

+ {#if showHeading} +

Finish setting up

+ {/if}

@@ -549,7 +814,7 @@ {#if row.status === 'done'} {:else if row.status === 'running'} - + {:else if row.status === 'failed'} {:else if row.status === 'unknown'} @@ -692,7 +957,18 @@

{/snippet} {#snippet detail()} - {#if b.occupiedBy} + {#if b.reusedFrom} + + now uses {b.reusedFrom} + + {#if b.stubKept} + + + some items still read {b.path} — fill it in too + + {/if} + {:else if b.occupiedBy} a {resourceTypeDisplayName(b.occupiedBy)} resource already holds this path — the project did not get this one @@ -719,15 +995,16 @@ {b.occupiedBy ? 'Resolve in the workspace' : 'Check the workspace'} + {:else if b.reusedFrom && !b.stubKept} + + Reused {:else} @@ -739,36 +1016,16 @@
{/if} - + {#if outstanding === 0} Everything this project needs is configured. Finish, and it is ready to run. - {:else if pendingTables.length > 0} - 0 - ? 'The project will not run without this' - : 'This could not be checked'} - size="xs" - > - {#if missingTables.length > 0} - The tables {missingTables.length === 1 ? 'this data table holds' : 'these data tables hold'} - do not exist, and the project's apps and flows read them. Every one of those fails as soon - as it opens. - {/if} - {#if uncheckedTables.length > 0} - {#if missingTables.length > 0}

{/if} - {uncheckedTables.length === 1 ? 'One data table is' : 'Some data tables are'} set up, but - {uncheckedTables.length === 1 ? 'its' : 'their'} schema could not be read, so whether the - project's tables are there is unknown. Check again once the database is reachable. - {/if} -
- {:else} + {:else if pendingTables.length === 0} The project's apps and flows will fail wherever they read a credential that is still missing. Everything else it imported works either way, and you can fill these in from the @@ -777,7 +1034,11 @@ {/if} {/if} -
+
{#if onBack} @@ -859,3 +1120,55 @@ void refreshBlanks()} /> + + + choosing !== undefined, + (v) => { + if (!v && !reusing) choosing = undefined + } + } +> + {#if choosing} + {@const forRow = choosing} + {@const existing = candidates[forRow.resourceType] ?? []} +
+

+ This workspace already has {existing.length} + {resourceTypeDisplayName(forRow.resourceType)} + {existing.length === 1 ? 'resource' : 'resources'}. Use one and this project's apps, flows + and triggers are pointed at it. +

+
+ + Service name, environment, resource attributes, metrics temporality and other + options are set with environment variables. + {/if}
{:else if setting.fieldType == 'otel_tracing_proxy'} @@ -846,9 +855,7 @@ {:else if setting.fieldType == 'seconds'}
@@ -863,6 +870,8 @@ {:else if setting.fieldType == 'ws_connectivity'} + {:else if setting.fieldType == 'instance_banner'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index a1a92acf09..42004aa698 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -14,6 +14,7 @@ import { sleep } from '$lib/utils' import { enterpriseLicense } from '$lib/stores' + import { isCloudHosted } from '$lib/cloud' import { createEventDispatcher } from 'svelte' import { setLicense } from '$lib/enterpriseUtils' @@ -100,7 +101,8 @@ otel: {}, indexer_settings: {}, critical_error_channels: [], - github_enterprise_app: {} + github_enterprise_app: {}, + instance_banner: {} } function applyFormDefaults(vals: Record): void { @@ -132,11 +134,15 @@ } applyFormDefaults(nvalues) - // Apply select/select_python defaults so initialValues matches what InstanceSetting's $effect does + // Apply declared defaults before snapshotting initialValues, so a default shows without + // marking the form dirty: a select's mirrors InstanceSetting's $effect, a boolean's is + // what its toggle shows while the key is unset. for (const category of settingsKeys) { for (const s of settings[category]) { if ( - (s.fieldType === 'select' || s.fieldType === 'select_python') && + (s.fieldType === 'select' || + s.fieldType === 'select_python' || + s.fieldType === 'boolean') && nvalues[s.key] == undefined && s.defaultValue ) { @@ -524,6 +530,10 @@ for (const category of settingsKeys) { const categorySettings = getSettingsForCategory(category) result[category] = categorySettings.some((s) => { + // A field the build never renders must not be able to block Save: off-cloud its + // value is unreachable, so an invalid one (from config sync, say) would leave the + // category permanently unsaveable with nothing on screen to fix. + if (s.cloudonly && !isCloudHosted()) return false if (s.isValid && !s.isValid(currentValues?.[s.key])) return true if (s.validate) { const errors = s.validate(currentValues?.[s.key]) @@ -1059,7 +1069,10 @@
  • instance base URL
  • login type usage (login type, count)
  • worker usage (worker, worker instance, vCPUs, memory)
  • -
  • user usage (author count, operator count)
  • +
  • user usage (author count, operator count, the distinct guests of the last 30 days, + the seats they add past the free allowance, and the workspaces that allow guests)
  • superadmin email addresses
  • development instance status
  • @@ -1069,13 +1082,27 @@
  • git sync repo count (sync vs promotion mode)
  • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, the plan tier and quota shown - when the execution meter is opened, and whether a variable whose value expired was a - secret, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your - deployed items use)
  • feature adoption (counts of which flow, script, trigger, worker and data table + features your deployed items use, including how many apps run sandboxed, how many data + tables exist per database kind, how many use migrations, and what references them)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code @@ -1118,17 +1145,34 @@
  • job usage (language, total duration, count)
  • login type usage (login type, count)
  • worker usage (worker, worker instance, vCPUs, memory)
  • -
  • user usage (author count, operator count)
  • +
  • user usage (author count, operator count, the distinct guests of the last 30 days, + the seats they add past the free allowance, and the workspaces that allow guests)
  • development instance status
  • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, the plan tier and quota shown - when the execution meter is opened, and whether a variable whose value expired was a - secret, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your - deployed items use)
  • feature adoption (counts of which flow, script, trigger, worker and data table + features your deployed items use, including how many apps run sandboxed, how many data + tables exist per database kind, how many use migrations, and what references them)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code @@ -1143,6 +1187,31 @@ description="Configure default timeouts and retention policies for job execution." link="https://www.windmill.dev/docs/advanced/instance_settings#jobs" /> + {:else if category == 'Service logs'} + + {#if !$values['object_store_cache_config']} +
    + + Instance object storage is not configured, so every server and worker keeps its log + files on its own disk. This page lists what each host wrote, but can only open the files + belonging to the replica serving the request — another host's are listed and not + readable — and a host's files go with it when it is replaced. Retention below still + governs the entries in the database and the files on disk. + +
    + {:else if !$enterpriseLicense} +
    + + Log files are uploaded to instance object storage, and the indexer that would ingest + them into the columnar store and delete each one afterwards is an enterprise feature. + Retention below expires the database entries and the local files; the uploaded copies + are only removed when Delete logs from s3 periodically is on under Object Storage. + +
    + {/if} {:else if category == 'Object Storage'} Value {#snippet headerAction()} -
    - -
    + {#if !disableExpand} +
    + +
    + {/if} {/snippet} diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index 71bd2fbb51..324bdea6c7 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -14,7 +14,6 @@ type WorkflowStatus, type OpenFlow } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { getViewToken } from '$lib/viewToken' import { WM_LOGS_SKIPPED } from '$lib/consts' import { getContext, onDestroy, tick, untrack } from 'svelte' @@ -22,6 +21,9 @@ import { sendUserToast } from '$lib/toast' import { DynamicInput, isScriptPreview } from '$lib/utils' import { getActiveReplay, getReplayStartTime } from './recording/replay.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() // Will be set to number if job is not a flow @@ -75,7 +77,7 @@ children }: Props = $props() - let workspace = $derived(workspaceOverride ?? $workspaceStore) + let workspace = $derived(workspaceOverride ?? $operatingWorkspace) let syncIteration: number = 0 let errorIteration = 0 diff --git a/frontend/src/lib/components/JobOtelTraces.svelte b/frontend/src/lib/components/JobOtelTraces.svelte index d57a7158e1..9332abdff1 100644 --- a/frontend/src/lib/components/JobOtelTraces.svelte +++ b/frontend/src/lib/components/JobOtelTraces.svelte @@ -1,9 +1,11 @@ + +
    +
    + {@render children()} +
    +
    diff --git a/frontend/src/lib/components/LabelsInput.svelte b/frontend/src/lib/components/LabelsInput.svelte index 933fe38ea1..cd85b9fd76 100644 --- a/frontend/src/lib/components/LabelsInput.svelte +++ b/frontend/src/lib/components/LabelsInput.svelte @@ -2,15 +2,30 @@ import Badge from './common/badge/Badge.svelte' import Button from './common/button/Button.svelte' import { Plus, Tag, X } from 'lucide-svelte' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { labels: string[] | undefined onchange?: () => void class?: string + /** Suggest the labels of this workspace rather than the active one, for an editor + * aimed elsewhere (the folder drawer opened from a cross-workspace picker). */ + workspace?: string + /** Text typed into the input but not yet added to `labels`. An editor with a Save + * button needs it: without it that text is invisible to the editor's dirty state, + * so it is silently dropped on close and cannot even enable Save on its own. */ + onPendingChange?: (pending: string) => void } - let { labels = $bindable(), onchange, class: clazz = '' }: Props = $props() + let { + labels = $bindable(), + onchange, + class: clazz = '', + workspace, + onPendingChange + }: Props = $props() let adding = $state(false) let inputValue = $state('') @@ -34,9 +49,13 @@ !(labels ?? []).includes(trimmedInput) ) + $effect(() => { + onPendingChange?.(adding ? trimmedInput : '') + }) + async function loadExistingLabels() { try { - const resp = await fetch(`/api/w/${$workspaceStore}/labels/list`) + const resp = await fetch(`/api/w/${workspace ?? $operatingWorkspace}/labels/list`) if (resp.ok) existingLabels = await resp.json() } catch {} } @@ -82,8 +101,15 @@ addLabel() // either "Create new" selected or free text } } else if (e.key === 'Escape') { + // Escape cancels the label, and nothing else. Left to bubble it also reaches + // whatever encloses us — a drawer or dialog closes on it, and one guarding on + // unsaved changes reads `pending` before this clears it, so it prompts to + // discard work this key just discarded. + e.preventDefault() + e.stopPropagation() inputValue = '' adding = false + onPendingChange?.('') } else if (e.key === 'ArrowDown') { e.preventDefault() const maxIdx = suggestions.length + (showCreateNew ? 1 : 0) - 1 @@ -100,6 +126,14 @@ if (adding) addLabel() }, 150) } + + /** Add whatever is typed but not yet committed, right now. Blur commits on a 150ms + * grace period, so a caller that reads `labels` in the same tick as the blur — a Save + * button, which blurs this input by being clicked — would miss the last label. + * `adding` is cleared here, so the pending timer then finds nothing to do. */ + export function flushPendingLabel(): void { + if (adding) addLabel() + }
    diff --git a/frontend/src/lib/components/LightweightResourcePicker.svelte b/frontend/src/lib/components/LightweightResourcePicker.svelte index bb0c21b4c1..14a7228ef5 100644 --- a/frontend/src/lib/components/LightweightResourcePicker.svelte +++ b/frontend/src/lib/components/LightweightResourcePicker.svelte @@ -1,6 +1,5 @@
    @@ -31,9 +32,11 @@
    -
    - - -
    + {#if showVersion} +
    + + +
    + {/if}
    diff --git a/frontend/src/lib/components/ModulePreview.svelte b/frontend/src/lib/components/ModulePreview.svelte index a9b769a4a5..be62557b81 100644 --- a/frontend/src/lib/components/ModulePreview.svelte +++ b/frontend/src/lib/components/ModulePreview.svelte @@ -21,6 +21,10 @@ class?: string onJobDone?: () => void hideRunButton?: boolean + /** Passed through to the form: the step whose agent form this preview accompanies. */ + openFieldsKey?: string + /** Passed through to the form: fields it must offer whatever the step holds. */ + runInputKeys?: readonly string[] } let { @@ -34,7 +38,9 @@ focusArg = undefined, class: className = '', onJobDone, - hideRunButton = false + hideRunButton = false, + openFieldsKey = undefined, + runInputKeys = undefined }: Props = $props() const { flowStore } = getContext('FlowEditorContext') @@ -85,5 +91,5 @@
    {/if} - +
  • diff --git a/frontend/src/lib/components/ModulePreviewForm.svelte b/frontend/src/lib/components/ModulePreviewForm.svelte index 88b99a05c5..093661c1e8 100644 --- a/frontend/src/lib/components/ModulePreviewForm.svelte +++ b/frontend/src/lib/components/ModulePreviewForm.svelte @@ -8,12 +8,17 @@ import { getContext, untrack } from 'svelte' import type { FlowEditorContext } from './flows/types' import { evalValue } from './flows/utils.svelte' + import { memoryPropertyFor } from './flows/flowInfers' import type { FlowModule } from '$lib/gen' import type { PickableProperties } from './flows/previousResults' import type SimpleEditor from './SimpleEditor.svelte' import { getResourceTypes } from './resourceTypesStore' import { twMerge } from 'tailwind-merge' - import { workspaceStore } from '$lib/stores' + import { AGENT_FIELDS, initialVisibleAgentFields } from './flows/agentFormFields' + import { openAgentFields } from './flows/content/AiAgentStepInputs.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { schema: Schema | { properties?: Record; required?: string[] } @@ -22,6 +27,12 @@ isValid?: boolean autofocus?: boolean focusArg?: string + /** Identifies the step whose agent form this one accompanies, so it can offer the fields that + * form has open. Same key `AiAgentStepInputs` is given. */ + openFieldsKey?: string + /** Fields to offer whatever the step holds, for a surface where nothing else can set them + * (`AGENT_EDITOR_RUN_INPUTS`). */ + runInputKeys?: readonly string[] } let { @@ -30,28 +41,70 @@ pickableProperties, isValid = $bindable(true), autofocus = false, - focusArg = undefined + focusArg = undefined, + openFieldsKey = undefined, + runInputKeys = [] }: Props = $props() const { stepsInputArgs, flowStateStore, flowStore, previewArgs, opWorkspace } = getContext('FlowEditorContext') - let opWs = $derived(opWorkspace?.() ?? $workspaceStore) + let opWs = $derived(opWorkspace?.() ?? $operatingWorkspace) let inputCheck: { [id: string]: boolean } = $state({}) $effect(() => { isValid = allTrue(inputCheck) ?? false }) + /** An agent asks for the same fields here that its own form shows: a setting the step leaves + * unset is not something a run needs told, and listing all eleven buries the message under the + * configuration. What the step configures stays, as it does on any other step, along with the + * rows its form has open — a field added there and left at its default reads as unset from the + * transforms alone, and this form has no add-field control to get it back. `runInputKeys` is + * for a surface whose form cannot open a row at all. A schema key the field registry doesn't + * know is kept, so a new one is never silently dropped. */ + let schemaKeys = $derived(Object.keys(schema?.properties ?? {})) + // A legacy memory kind this step still holds stays one of the options, or the one-of field would + // turn the test run's memory off. + let isAgent = $derived((mod.value as { type?: string })?.type === 'aiagent') + + let visibleKeys = $derived.by(() => { + const all = schemaKeys + if ((mod.value as { type?: string })?.type !== 'aiagent') return all + const transforms = (mod.value as { input_transforms?: Record }) + ?.input_transforms + const visible = initialVisibleAgentFields(transforms, schema?.properties) + for (const key of openAgentFields(openFieldsKey)) visible.add(key) + for (const key of runInputKeys) visible.add(key) + const known = new Set(AGENT_FIELDS.map((f) => f.key)) + // Listed in the agent form's order rather than the schema's, so the two read the same. + const position = new Map(AGENT_FIELDS.map((f, i) => [f.key, i])) + return all + .filter((key) => !known.has(key) || visible.has(key)) + .sort((a, b) => (position.get(a) ?? Infinity) - (position.get(b) ?? Infinity)) + }) + let keys: string[] = $state([]) $effect(() => { - let lkeys = Object.keys(schema?.properties ?? {}) + let lkeys = visibleKeys if (schema?.properties && JSON.stringify(lkeys) != JSON.stringify(keys)) { keys = lkeys - untrack(() => stepsInputArgs?.removeExtraKey(mod.id, keys)) + // Pruned against the schema rather than against what is shown. What a run was given for a + // field lives only here, so dropping it when the field merely stops being displayed would + // discard it: an agent hides the settings its step leaves unset, and clearing one in the + // Inputs tab hides it. + untrack(() => stepsInputArgs?.removeExtraKey(mod.id, schemaKeys)) } }) + /** Whether re-evaluating has anything to restore. A field the step configures nothing for + * evaluates to blank, so the control would only clear what was typed to run with. */ + function hasConfiguredInput(argName: string): boolean { + const transform = (mod.value as any)?.input_transforms?.[argName] + if (!transform) return false + return transform.type === 'javascript' ? !!transform.expr : transform.value !== undefined + } + function plugIt(argName: string) { stepsInputArgs?.setEvaluatedStepArg( mod.id, @@ -139,7 +192,12 @@ (v) => stepsInputArgs?.setStepInputArgs(mod.id, argName, v) } type={schema.properties[argName].type} - oneOf={schema.properties[argName].oneOf} + oneOf={isAgent && argName === 'memory' + ? memoryPropertyFor( + schema.properties[argName], + stepsInputArgs?.getStepInputArgs(mod.id, argName) + )?.oneOf + : schema.properties[argName].oneOf} required={schema?.required?.includes(argName)} pattern={schema.properties[argName].pattern} bind:editor={editor[argName]} @@ -158,7 +216,7 @@ workspace={opWs} > {#snippet fieldHeaderActions()} - {#if stepsInputArgs?.isArgManuallySet(mod.id, argName)} + {#if stepsInputArgs?.isArgManuallySet(mod.id, argName) && hasConfiguredInput(argName)}
    - {/each} +{#if options.length > 0} +
    + {#each options as option (option)} + + {/each} +
    + Custom scopes {/if} +{#each custom as v, i (i)} +
    + setRow(i, e.currentTarget.value) }} + /> +
    +{/each} +
    - - ({(scopes ?? []).length} item{(scopes ?? []).length > 1 ? 's' : ''}) - + {#if custom.length > 0} + + ({custom.length} item{custom.length > 1 ? 's' : ''}) + + {/if}
    diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index c7f7065565..9ddd7f7fdc 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -1,3 +1,42 @@ + +
    - {#if loading} + {#snippet action()} + + {#snippet children({ item })} + {#each Object.keys(WINDOWS) as key (key)} + + {/each} + {/snippet} + + {/snippet} + + {#if metrics.error} + {metrics.error.message} + {:else if metrics.current === undefined} - {:else if noMetrics} -

    No jobs delayed by more than 3 seconds in the last 14 days

    + {:else if metrics.current.tags.length === 0} +

    + No jobs delayed by more than 3 seconds in the last {WINDOWS[windowKey].label} +

    {:else}
    - {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { - title: { - display: true, - text: 'count' - } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' - }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } - } - } - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', - title: { - display: true, - text: 'delay (s)' - }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) - } - } - } - }} - /> - {/if} + + - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. + Only tags with jobs delayed by more than 3 seconds in this window are included. At wide + windows a line shows the highest value of each time slot, so short spikes stay visible.
    {/if} diff --git a/frontend/src/lib/components/QueuePosition.svelte b/frontend/src/lib/components/QueuePosition.svelte index 135feae899..61245613d6 100644 --- a/frontend/src/lib/components/QueuePosition.svelte +++ b/frontend/src/lib/components/QueuePosition.svelte @@ -1,8 +1,10 @@ + +
    + {#snippet action()} +
    + Tag + Waiting + Next job's wait + Running + Workers + + + + {#each rows as s (s.tag)} + + {s.tag} + {s.waiting} + + {s.delay === undefined ? '-' : msToReadableTime(s.delay * 1000, 0)} + + {s.running} + + {#if unserved(s)} + + + No worker currently pulls this tag + + {:else} + {s.workers} + {/if} + + + {/each} + + + {/if} + diff --git a/frontend/src/lib/components/ResourceDescriptionField.svelte b/frontend/src/lib/components/ResourceDescriptionField.svelte new file mode 100644 index 0000000000..2f065176dc --- /dev/null +++ b/frontend/src/lib/components/ResourceDescriptionField.svelte @@ -0,0 +1,59 @@ + + + +
    +

    {label} + + {#if canWrite} +

    + {#if canWrite && editing} +
    +
    GH Markdown
    + +
    + {:else if description == undefined || description == ''} +
    No description provided
    + {:else} + + {/if} +
    diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 367888655f..9847454aca 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -1,19 +1,31 @@
    + + {#if draftConflict && !onDraftConflictChange} + void resolveDraftConflict(false)} + onOverwrite={() => void resolveDraftConflict(true)} + /> + {/if} + {#if otherDirty.length > 0} You are going to edit the value in: {otherDirty.join(', ')} {/if} - {#if current} + {#if holdsCredential && selected} + +
    +
    + The URL carries no credential. Windmill stores the token and renews it before it + expires. + {#if urlDirty} + Save your URL change to replace the token. + {/if} +
    + +
    +
    + {/if} + + + {#if current && can_write !== undefined} {#key current} current!.path, setPath} @@ -391,7 +744,7 @@ bind:args={current.args} bind:wsSpecific={current.wsSpecific} bind:isValid - bind:viewJsonSchema + bind:viewJsonSchema={() => viewJsonSchema ?? false, (v) => (viewJsonSchema = v)} bind:jsonError {initialPath} {hidePath} @@ -404,6 +757,7 @@ {resourceToEdit} onLoadResourceType={() => resourceTypeResource.refetch()} workspace={selected} + actingUser={acting.in(selected) ?? null} /> {/key} {/if} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 296ab693a8..ce4d1eea5a 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -5,12 +5,14 @@ import { History, Loader2, Save } from 'lucide-svelte' import WsSpecificVersions from './WsSpecificVersions.svelte' - import { userStore, workspaceStore } from '$lib/stores' import { isOwner } from '$lib/utils' + import { useActingUser } from '$lib/actingUser.svelte' import LocalDraftBanner from './LocalDraftBanner.svelte' + import DraftConflictAlert from './DraftConflictAlert.svelte' import OpenInSessionButton from './sessions/OpenInSessionButton.svelte' import { clearPageDrawerAnchor, + handOffPageDrawer, pageDrawerSessionSource, setPageDrawerAnchor } from './sessions/pageDrawerSession' @@ -18,19 +20,34 @@ import ResourceVersionHistory from './ResourceVersionHistory.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' + import { loadResourceTypeDisplayName } from './displayNameLoaders' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { workspace = undefined, disableChatOffset = false, + inline = false, + onClose = undefined, onRestored = undefined, onSaved = undefined }: { + /** Workspace this drawer acts in. Optional; the navigation workspace is substituted + * once, at `effectiveWorkspace`, and nowhere else in this file. */ workspace?: string disableChatOffset?: boolean + /** Render in place, filling the parent, with no drawer or close button — for a host + * that gives the editor a whole pane. Saving and restoring then leave it open: the + * host remounts it on what was written. */ + inline?: boolean + /** With `inline`, closes whatever hosts the editor; the header has a close button only when set. */ + onClose?: () => void onRestored?: () => void /** Fires after Save has written, for a caller showing state derived from the - * resource — `onRestored` only covers restoring an old version. */ - onSaved?: () => void + * resource — `onRestored` only covers restoring an old version. `path` is where the + * resource now lives in this drawer's workspace, undefined when the save failed. */ + onSaved?: (path: string | undefined) => void } = $props() let drawer: Drawer | undefined = $state() @@ -41,34 +58,60 @@ let resourceEditor: | { - save: () => void + save: () => Promise + pathIn: (ws: string) => string | undefined localDraftDeployed: () => unknown localDraftCurrent: () => unknown discardLocalDraft: () => void + endEditingSession: () => void + resolveDraftConflictFromBanner: (keepMine: boolean) => void } | undefined = $state(undefined) let hasLocalDraft = $state(false) + let draftConflict = $state({ conflicted: false, busy: false }) let canWriteSelected = $state(true) let path: string | undefined = $state(undefined) let selected: string | undefined = $state(undefined) + let viewJsonSchema = $state(false) - let effectiveWorkspace = $derived(workspace ?? $workspaceStore!) + let effectiveWorkspace = $derived(workspace ?? $operatingWorkspace!) // The editor renders whichever workspace-specific variant `selected` points at, so history has // to follow it too — otherwise a restore would write over the variant the user is not looking at. let historyWorkspace = $derived(selected ?? effectiveWorkspace) - // Clearing is irreversible and the backend gates it on ownership, not write access. $userStore - // describes the user in the workspace they are signed into, so it can only answer for that one: - // history pointed anywhere else — a ws-specific variant, or an explicit `workspace` prop — gets - // no Clear button rather than a verdict computed from the wrong membership. - let canClearSelected = $derived( - historyWorkspace === $workspaceStore && isOwner(path ?? '', $userStore, $workspaceStore) - ) + // Gated on `path`: this drawer outlives every resource it opens, so there is nothing to + // answer about until one is open. + const historyUser = useActingUser(() => (path ? historyWorkspace : undefined)) + // Clearing is irreversible and the backend gates it on ownership, not write access, so the + // verdict has to come from the membership `historyWorkspace` knows about. An unresolved + // user gets no Clear button rather than one computed from another workspace's rights. + let canClearSelected = $derived(isOwner(path ?? '', historyUser.current, historyWorkspace)) - export async function initEdit(p: string): Promise { + // A close reaches `on:close` on a later flush, by which point a caller that closed this drawer to + // open another editor has already anchored the new one. Clearing then would strip that anchor. + let keepAnchorOnClose = false + + /** Shut this drawer without going through its own close button, for a caller opening the other + * editor over the same list. `keepAnchor` when that caller anchors what it opens instead. */ + export function close(opts?: { keepAnchor?: boolean }): void { + keepAnchorOnClose = opts?.keepAnchor ?? false + drawer?.closeDrawer?.() + } + + /** `json` opens on the JSON editor instead of the resource type's form. For a type with a + * dedicated editor elsewhere: the generic form would render its configuration field by field, + * and materialize a default into every one the value leaves out. */ + export async function initEdit(p: string, opts?: { json?: boolean }): Promise { + if (handOffPageDrawer(RESOURCES_PATH, p)) return + // A `close({ keepAnchor })` on an already-closed drawer emits no close event, so the flag + // would still be standing when the next drawer session ends and would swallow that one's + // anchor clear. Every session starts having to clear its own. + keepAnchorOnClose = false + historyUser.forgetFailures() resource_type = undefined path = p selected = effectiveWorkspace + viewJsonSchema = opts?.json ?? false drawer?.openDrawer?.() setPageDrawerAnchor(RESOURCES_PATH, p) } @@ -77,10 +120,18 @@ resourceType: string, nDefaultValues?: Record ): Promise { + keepAnchorOnClose = false + historyUser.forgetFailures() path = undefined resource_type = resourceType defaultValues = nDefaultValues selected = effectiveWorkspace + // This drawer outlives what it opens on, so the view has to be set by every entry point + // rather than left where the last one put it: a new resource is a typed form, whoever was + // looking at JSON before. + viewJsonSchema = false + // The title names the type, whose row nothing else on the page may have read. + void loadResourceTypeDisplayName(effectiveWorkspace, resourceType) drawer?.openDrawer?.() } @@ -93,16 +144,46 @@ ) - clearPageDrawerAnchor(RESOURCES_PATH)} -> +{#if inline} + + {#if path !== undefined || resource_type !== undefined} + {@render content()} + {/if} +{:else} + { + // The editor outlives this drawer, so tell it the session is over: a conflict resolution + // still in flight must not land on whatever the next opening shows. + resourceEditor?.endEditingSession?.() + if (keepAnchorOnClose) { + keepAnchorOnClose = false + return + } + clearPageDrawerAnchor(RESOURCES_PATH) + }} + > + {@render content()} + +{/if} + +{#snippet content()} { + // Inline has no drawer to emit a close, so the session ends here instead. + if (inline) { + resourceEditor?.endEditingSession?.() + onClose?.() + } else { + drawer?.closeDrawer() + } + }} > {#snippet titleExtra()} {#if mode == 'new' && resource_type} @@ -116,16 +197,25 @@ {path} {resource_type} {defaultValues} - {workspace} + workspace={effectiveWorkspace} on:refresh bind:this={resourceEditor} bind:canSave bind:selected + bind:viewJsonSchema onDraftStateChange={(v) => (hasLocalDraft = v)} + onDraftConflictChange={(v) => (draftConflict = v)} onCanWriteChange={(v) => (canWriteSelected = v)} /> {/await} {#snippet banner()} + {#if draftConflict.conflicted} + resourceEditor?.resolveDraftConflictFromBanner?.(false)} + onOverwrite={() => resourceEditor?.resolveDraftConflictFromBanner?.(true)} + /> + {/if} @@ -172,7 +262,7 @@ {/snippet} - +{/snippet} diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index cddd83aae7..1d4f091236 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -1,4 +1,5 @@ - - diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 5006e2df0b..ce1af47118 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -7,7 +7,9 @@ import Select from './select/Select.svelte' import { FileUp } from 'lucide-svelte' import { SettingService } from '$lib/gen' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() import { resource } from 'runed' interface Props { @@ -54,7 +56,7 @@ * asset lives in a different workspace than this picker was mounted for. */ let workspaceOverride: string | undefined = $state(undefined) let effectiveWorkspace = $derived(workspaceOverride ?? workspace) - let ws = $derived(effectiveWorkspace ?? $workspaceStore) + let ws = $derived(effectiveWorkspace ?? $operatingWorkspace) let uploadModalOpen = $state(false) let allFilesByKey: Record< diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 5737744034..1bf1b4358a 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -10,7 +10,6 @@ Trash, MoveRight } from 'lucide-svelte' - import { workspaceStore } from '$lib/stores' import { CancelablePromise, HelpersService, @@ -48,6 +47,9 @@ import FileUploadModal from './common/fileUpload/FileUploadModal.svelte' import S3FilePreview from './S3FilePreview.svelte' import { twMerge } from 'tailwind-merge' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let deletionModalOpen = $state(false) let fileDeletionInProgress = $state(false) @@ -137,7 +139,7 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) diff --git a/frontend/src/lib/components/S3FilePreview.svelte b/frontend/src/lib/components/S3FilePreview.svelte index 62c95581e7..411592b936 100644 --- a/frontend/src/lib/components/S3FilePreview.svelte +++ b/frontend/src/lib/components/S3FilePreview.svelte @@ -8,7 +8,6 @@ // need to hand it a `fileKey`. CSV separator/header are local state so // the user can re-preview the same file with different parsing flags. import { FileX2, Loader2 } from 'lucide-svelte' - import { workspaceStore } from '$lib/stores' import { HelpersService, type CancelablePromise, @@ -20,6 +19,9 @@ import { displayDate, displaySize, emptyString } from '$lib/utils' import { twMerge } from 'tailwind-merge' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { fileKey: string | undefined @@ -115,7 +117,7 @@ // existence after an upstream run completes — moving from the // "not yet materialized" empty state to the actual preview without // requiring the user to re-click the asset. - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) $effect(() => { const key = fileKey diff --git a/frontend/src/lib/components/SaveInputsButton.svelte b/frontend/src/lib/components/SaveInputsButton.svelte index 91b583e0da..8d32b01163 100644 --- a/frontend/src/lib/components/SaveInputsButton.svelte +++ b/frontend/src/lib/components/SaveInputsButton.svelte @@ -1,12 +1,14 @@ + + diff --git a/frontend/src/lib/components/Section.svelte b/frontend/src/lib/components/Section.svelte index f1425c51b7..badc7c9531 100644 --- a/frontend/src/lib/components/Section.svelte +++ b/frontend/src/lib/components/Section.svelte @@ -102,7 +102,7 @@ transition:slide={animate || collapsable ? { duration: 200 } : { duration: 0 }} > {#if description} -
    {@html description}
    +
    {@html description}
    {/if}
    diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index bad18661b6..16bdd7d2ad 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -2,7 +2,7 @@ import { createBubbler, preventDefault } from 'svelte/legacy' const bubble = createBubbler() - import { IndexSearchService, ServiceLogsService } from '$lib/gen' + import { IndexSearchService, ServiceLogsService, type LogSearchHit } from '$lib/gen' import TimeframeSelect, { serviceLogsTimeframes, @@ -253,39 +253,50 @@ try { let res = '' log.split('\n').forEach((line) => { + // A file can hold both formats: the ones written before the layer + // switched to JSON, and panics or subprocess output that was never + // JSON to begin with. Those lines pass through as they are rather + // than being dropped, which would render the file blank. + let obj: any = undefined if (line.startsWith('{') && line.endsWith('}')) { - let obj = JSON.parse(line) - if (typeof obj == 'object') { - let nl = '' - if (obj['timestamp']) { - nl += obj['timestamp'] + ' ' - } - if (obj['level']) { - let lvl = obj['level'] - if (lvl == 'ERROR') { - nl += '\x1b[31mERROR\x1b[0m ' - } else if (lvl == 'INFO') { - nl += '\x1b[32mINFO\x1b[0m ' - } else { - nl += obj['level'] + ' ' - } - } - if (obj['message']) { - nl += obj['message'] + ' ' - } - delete obj['timestamp'] - delete obj['level'] - delete obj['message'] - Object.keys(obj).forEach((key) => { - nl += - key + - '=' + - (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + - ' ' - }) - res += nl + '\n' + try { + obj = JSON.parse(line) + } catch { + obj = undefined } } + if (obj === null || typeof obj !== 'object') { + res += line + '\n' + } else { + let nl = '' + if (obj['timestamp']) { + nl += obj['timestamp'] + ' ' + } + if (obj['level']) { + let lvl = obj['level'] + if (lvl == 'ERROR') { + nl += '\x1b[31mERROR\x1b[0m ' + } else if (lvl == 'INFO') { + nl += '\x1b[32mINFO\x1b[0m ' + } else { + nl += obj['level'] + ' ' + } + } + if (obj['message']) { + nl += obj['message'] + ' ' + } + delete obj['timestamp'] + delete obj['level'] + delete obj['message'] + Object.keys(obj).forEach((key) => { + nl += + key + + '=' + + (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + + ' ' + }) + res += nl + '\n' + } }) return res @@ -294,6 +305,23 @@ } } + // A hit is one log line with its fields already separated, so rendering it is + // formatting rather than parsing — there is no JSON to prettify and no + // snippet to highlight. + function renderHit(hit: LogSearchHit): string { + const level = + hit.level === 'ERROR' + ? '\x1b[31mERROR\x1b[0m' + : hit.level === 'WARN' + ? '\x1b[33mWARN\x1b[0m' + : hit.level === 'INFO' + ? '\x1b[32mINFO\x1b[0m' + : hit.level + return [hit.ts, level, hit.message, hit.target ? `target=${hit.target}` : ''] + .filter(Boolean) + .join(' ') + } + let logs: any = $state() let debounceTimeout: number | undefined = undefined @@ -399,7 +427,9 @@ ) { const res = await ServiceLogsService.getLogFile({ path: `${hostname}/${path}` }) - content = processLogWithJsonFmt(ansi_up.ansi_to_html(res), jsonFmt) + // Prettify first: it emits its own ANSI for the level, which converting + // beforehand would leave in the output as literal escapes. + content = ansi_up.ansi_to_html(processLogWithJsonFmt(res, jsonFmt)) hitLineNumber = lineNumber logDrawerOpen = true @@ -687,23 +717,19 @@
    {:else if logs != undefined}
    - {#each logs.hits as { snippet_fragment, snippet_highlighted, document }} + + {#each logs.hits ?? [] as hit, i (`${i}:${hit.file_path}:${hit.line_no}`)} { - let logLineNumber = document.line_number[0] - let logFile = document.file_name[0] - let host = document.host[0] - let jsonFmt = document.json_fmt[0] - seeLogContext(logLineNumber, logFile, host, jsonFmt) - }} + content={renderHit(hit)} + highlighted={[]} + onClick={() => seeLogContext(hit.line_no, hit.file_path, hit.host, true)} /> {/each} - {#if logs.hits.length === 0} + {#if (logs.hits ?? []).length === 0}
    No logs
    {/if} - {#if logs.hits.length === 1000} + {#if (logs.hits ?? []).length === 1000}
    Older matches were truncated from this search, try refining your filters to get more precise results. diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 29da96cf98..471ed74099 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -252,7 +252,7 @@ {/if}
    Extra permissions ({acls?.length ?? 0})Extra members ({acls?.length ?? 0}) {#if linkedVarPaths.length > 0}
    @@ -299,7 +299,7 @@ size="lg" variant="accent" disabled={!newOwner} - on:click={() => addAcl(newOwner, write)}>Add permission addAcl(newOwner, write)}>Add member
    {/if} @@ -307,7 +307,7 @@ {#snippet headerRow()}
    - + diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 2764b130e2..8133494f18 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -63,7 +63,14 @@ /** Trailing debounce window (ms) on Monaco's onDidChangeModelContent. */ const CHANGE_TIMEOUT = 200 + /** Gap between the line numbers and the first character. Zero puts them flush, + * so a two-digit line reads as one token with the code. */ + const LINE_DECORATIONS_WIDTH = 6 + let changeTimeoutId: number | undefined = undefined + // Monaco fires onDidChangeModelContent synchronously from within `setValue`, so without + // this an authoritative overwrite reads as a user edit on the `input` event. + let applyingCode = false let divEl: HTMLDivElement | null = null let editor = $state(null) @@ -74,6 +81,10 @@ let width = $state(0) let initialized = $state(false) let placeholderVisible = $state(false) + // Monaco's content origin. The placeholder is a plain overlay on the editor container, so + // without these it sits over the line-number gutter and off the line-1 baseline. + let contentLeft = $state(0) + let contentLineHeight = $state(0) let mounted = $state(false) let valueAfterDispose: string | undefined = undefined @@ -105,7 +116,8 @@ minHeight = 1000, renderLineHighlight = 'none', suggestion, - leadingChangeSync = false + leadingChangeSync = false, + lineNumbersMinChars = 3 }: { lang: string code?: string @@ -142,6 +154,9 @@ * `code`; leave it off where each extra sync costs work downstream (an app * code input feeding an autoRefresh runnable re-runs a job per sync). */ leadingChangeSync?: boolean + /** Width of the line-number gutter, in characters. Same name, and same + * default, as `Editor`, so the two render line numbers alike. */ + lineNumbersMinChars?: number } = $props() let yPadding = MONACO_Y_PADDING @@ -179,7 +194,12 @@ if (ncode != code) { code = ncode } - editor?.setValue(ncode) + applyingCode = true + try { + editor?.setValue(ncode) + } finally { + applyingCode = false + } // setValue emits a change event of its own; drop the burst it opens so an edit // made right after an authoritative overwrite still counts as a leading change. cancelPendingChanges() @@ -300,10 +320,12 @@ if (model.getLanguageId() !== lang) { const currentCode = model.getValue() const uri = `file:///${hash}.${langToExt(lang)}` - const oldModel = model - const newModel = meditor.createModel(currentCode, lang, mUri.parse(uri)) - editor?.setModel(newModel) - oldModel.dispose() + // The old model goes first: `langToExt` maps anything it does not know to + // `unknown`, so the new uri is usually the one this model already holds, + // and creating over an occupied uri throws ("model already exists"). + editor?.setModel(null) + model.dispose() + editor?.setModel(meditor.createModel(currentCode, lang, mUri.parse(uri))) } // Update editor options for suggestions, validation decorations, and line numbers @@ -322,8 +344,8 @@ snippetsPreventQuickSuggestions: disableSuggestions }, lineNumbers: hideLineNumbers ? 'off' : 'on', - lineDecorationsWidth: hideLineNumbers ? 0 : 6, - lineNumbersMinChars: hideLineNumbers ? 0 : 2, + lineDecorationsWidth: hideLineNumbers ? 0 : LINE_DECORATIONS_WIDTH, + lineNumbersMinChars: hideLineNumbers ? 0 : lineNumbersMinChars, // Hide validation squiggles and decorations renderValidationDecorations: disableLinting ? 'off' : 'on', // Hide the validation margin indicators @@ -385,8 +407,11 @@ ...(yPadding !== undefined ? { padding: { bottom: yPadding, top: yPadding } } : {}), readOnly, renderLineHighlight, - lineDecorationsWidth: 0, - lineNumbersMinChars: 2, + // Same conditional as `updateModelAndOptions`: created correct rather than + // created wide and narrowed a tick later, which a caller hiding the gutter + // would see as a flash of indent. + lineDecorationsWidth: hideLineNumbers ? 0 : LINE_DECORATIONS_WIDTH, + lineNumbersMinChars: hideLineNumbers ? 0 : lineNumbersMinChars, fontSize: fontSize, quickSuggestions: disableSuggestions ? { other: false, comments: false, strings: false } @@ -454,6 +479,12 @@ changeTimeoutId = undefined updateCode() }, CHANGE_TIMEOUT) + // `change` trails the buffer by CHANGE_TIMEOUT, too late for a consumer that has to + // know the moment the buffer stopped being the one it wrote. `input` says only that, + // carrying no value: read `getCode()` for what is on screen. + if (!applyingCode) { + dispatch('input') + } if (leading) { updateCode() } @@ -533,6 +564,13 @@ } if (placeholder) { + const syncPlaceholderOrigin = () => { + if (!editor) return + contentLeft = editor.getLayoutInfo().contentLeft + contentLineHeight = editor.getOption(meditor.EditorOption.lineHeight) + } + syncPlaceholderOrigin() + editor.onDidLayoutChange(syncPlaceholderOrigin) editor.onDidChangeModelContent(() => { if (!editor) return const value = editor.getValue() @@ -755,9 +793,10 @@ {#if placeholder}
    {@html placeholder}
    diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index e19da629bd..d58ac65898 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -3,7 +3,6 @@ import Button from './common/button/Button.svelte' import { runScriptAndPollResult } from './jobs/utils' import { writingJobOptions } from './jobs/writingJob' - import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { untrack } from 'svelte' import { getLanguageByResourceType } from './apps/components/display/dbtable/utils' @@ -14,6 +13,9 @@ import { wrapDucklakeQuery } from './ducklake' import { splitSqlStatements, pruneComments } from './sqlDdl' import DdlMigrationGuard from './DdlMigrationGuard.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() type Props = { input: DbInput @@ -39,7 +41,7 @@ workspace = undefined, tag = undefined }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) let dbType = $derived(getDbType(input)) // A datatable REPL targets `datatable://`; surface DDL statements as @@ -103,14 +105,21 @@ // We don't always put the fix by default for row ordering concerns let transformedCode = code if (doPostgresRowToJsonFix) { - transformedCode = statements - .map((statement) => { - if (READ_OPS.some((op) => statement.trim().toUpperCase().startsWith(op))) { - return `SELECT row_to_json(__t__) FROM (${statement}) __t__` - } - return statement - }) - .join(';') + // Rebuilt from the pruned statements, which drops the leading comment block — and + // with it the `-- role ` annotation that decides which login the query runs + // as. Carry it over, or the retry connects as the data table's default role and a + // query the first attempt was denied succeeds on the second. + const leadingAnnotations = code.match(/^(?:[^\S\n]*\n|[^\S\n]*--[^\n]*\n)*/)?.[0] ?? '' + transformedCode = + leadingAnnotations + + statements + .map((statement) => { + if (READ_OPS.some((op) => statement.trim().toUpperCase().startsWith(op))) { + return `SELECT row_to_json(__t__) FROM (${statement}) __t__` + } + return statement + }) + .join(';') } const dbArg = getDatabaseArg(input) @@ -216,5 +225,10 @@ {#if datatableName && ws} - + {/if} diff --git a/frontend/src/lib/components/Star.svelte b/frontend/src/lib/components/Star.svelte index dd91f671c8..4f0f169109 100644 --- a/frontend/src/lib/components/Star.svelte +++ b/frontend/src/lib/components/Star.svelte @@ -9,9 +9,10 @@ kind: FavoriteKind summary?: string workspaceId?: string + size?: number } - let { path, kind, workspaceId, summary }: Props = $props() + let { path, kind, workspaceId, summary, size = 16 }: Props = $props() let buttonHover = $state(false) let starred = $derived(favoriteManager.isStarred(path, kind)) @@ -31,14 +32,14 @@ > {#if starred} {#if buttonHover} - + {:else} - + {/if} {:else} {/if} diff --git a/frontend/src/lib/components/SummaryPathDisplay.svelte b/frontend/src/lib/components/SummaryPathDisplay.svelte index 790ca5b0f8..22925e98e7 100644 --- a/frontend/src/lib/components/SummaryPathDisplay.svelte +++ b/frontend/src/lib/components/SummaryPathDisplay.svelte @@ -4,13 +4,21 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import Path from '$lib/components/Path.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { updateItemPathAndSummary, checkFlowOnBehalfOf } from './moveRenameManager' import Label from './Label.svelte' import LabelsInput from './LabelsInput.svelte' import InheritedLabels from './InheritedLabels.svelte' import Badge from './common/badge/Badge.svelte' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) interface Props { summary?: string @@ -36,7 +44,11 @@ let editPath = $state('') let dirtyPath = $state(false) let popoverOpen = $state(false) - let own = $state(false) + let ownPath = $state(undefined) + // Derived: ownership answers about the operating workspace, whose user resolves asynchronously. + const own = $derived( + ownPath === undefined ? false : isOwner(ownPath, actingUser, $operatingWorkspace) + ) let onBehalfOfEmail = $state(undefined) let summaryInput: ReturnType | undefined = $state() let labelsDirty = $state(false) @@ -47,10 +59,10 @@ editSummary = summary ?? '' editPath = path ?? '' labelsDirty = false - own = isOwner(path ?? '', $userStore, $workspaceStore) + ownPath = path ?? '' onBehalfOfEmail = undefined - if (kind === 'flow' && $workspaceStore && path) { - checkFlowOnBehalfOf($workspaceStore, path).then((email) => { + if (kind === 'flow' && $operatingWorkspace && path) { + checkFlowOnBehalfOf($operatingWorkspace, path).then((email) => { onBehalfOfEmail = email }) } @@ -63,7 +75,7 @@ try { await updateItemPathAndSummary({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, kind, initialPath, newPath, diff --git a/frontend/src/lib/components/SuperadminSettings.svelte b/frontend/src/lib/components/SuperadminSettings.svelte index 316a6237cd..dfe20ae054 100644 --- a/frontend/src/lib/components/SuperadminSettings.svelte +++ b/frontend/src/lib/components/SuperadminSettings.svelte @@ -89,7 +89,7 @@ } - + {#snippet titleExtra()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index e74154b406..0fd7991597 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -1,5 +1,6 @@ - clearPageDrawerAnchor(VARIABLES_PATH)}> +{#if inline} + {@render content()} +{:else} + { + endEditingSession() + clearPageDrawerAnchor(VARIABLES_PATH) + }} + > + {@render content()} + +{/if} + +{#snippet content()} { + // Inline has no drawer to emit a close, so the session ends here instead. + if (inline) { + endEditingSession() + onClose?.() + } else { + drawer?.closeDrawer() + } + }} > {#snippet banner()} + {#if draftConflict} + void resolveDraftConflict(false)} + onOverwrite={() => void resolveDraftConflict(true)} + /> + {/if} {/snippet}
    - {#if !can_write} + {#if can_write === false} You only have read access to this resource and cannot edit it @@ -363,7 +557,9 @@ {/if} - {#if current} + + {#if current && can_write !== undefined} {#key current} {/key} {/if} @@ -399,4 +596,4 @@ {/snippet} - +{/snippet} diff --git a/frontend/src/lib/components/VariableForm.svelte b/frontend/src/lib/components/VariableForm.svelte index af8bb01919..5c1c55ae36 100644 --- a/frontend/src/lib/components/VariableForm.svelte +++ b/frontend/src/lib/components/VariableForm.svelte @@ -10,12 +10,15 @@ import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import { Loader2, RotateCcw } from 'lucide-svelte' import autosize from '$lib/autosize' - import { userStore, workspaceStore } from '$lib/stores' + import { type UserExt } from '$lib/stores' import { isOwner } from '$lib/utils' import { isEncryptedDraftValue } from '$lib/encryptedDraft' import EncryptedDraftField from './EncryptedDraftField.svelte' import DateTimeInput from './DateTimeInput.svelte' import { base } from '$lib/base' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Variable { value: string @@ -37,6 +40,11 @@ onLoadSecret?: () => void /** Workspace the path is validated against; defaults to the nav workspace. */ workspace?: string | undefined + /** The user acting in `workspace`, resolved by the editor above. `undefined` while + * `null` while that lookup is pending or after it failed: every check below then + * refuses, rather than answering with the navigation user's rights in another + * workspace. */ + actingUser: UserExt | null } let { @@ -51,10 +59,11 @@ can_write, edit, onLoadSecret, - workspace = undefined + workspace = undefined, + actingUser }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) // Enabling seeds a date rather than revealing an empty picker, mirroring how // `ScriptAdvancedSettings` seeds a timeout of 300s. An empty DateTimeInput renders a time @@ -86,13 +95,14 @@
    @@ -104,7 +114,7 @@ edit && !hasStagedValue && onLoadSecret?.()} bind:checked={variable.is_secret} - disabled={edit && ($userStore?.operator || isEncryptedDraftValue(variable.value))} + disabled={edit && (!actingUser || actingUser.operator || isEncryptedDraftValue(variable.value))} /> {#if variable.is_secret} @@ -146,7 +156,7 @@ > Reset - {:else if $userStore?.operator} + {:else if actingUser?.operator}
    Operators cannot load secret value
    {:else}
    + + + + + + + Question + Expected + + + + + {#each pageCases as c (c.id)} + + + + (c.input = { ...(c.input ?? {}), user_message: v })} + /> + + + setExpected(c, v)} + /> + + + + +{/if} diff --git a/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte b/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte index 3da84cf1ad..cbdf05658c 100644 --- a/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte +++ b/frontend/src/lib/components/aiEvals/EvalDatasetDrawer.svelte @@ -26,8 +26,7 @@ onRenamed, onDeleted, onCasesChanged, - onScorersChanged, - onClosed + onScorersChanged }: { workspace: string | undefined /** The agent the dataset is named after and belongs to. */ @@ -45,8 +44,6 @@ onDeleted: (path: string) => void | Promise onCasesChanged: () => void | Promise onScorersChanged: () => void | Promise - /** The drawer is done, whether it saved anything or not. */ - onClosed?: () => void } = $props() let drawer: Drawer | undefined = $state() @@ -92,11 +89,6 @@ let storedIds = $state>(new Set()) let removingCase = $state(undefined) - /** The grid, so an open cell can be committed before the list is read to write it. */ - let casesGrid: EvalCasesGrid | undefined = $state() - /** A cell is open in the grid: an edit the list does not hold yet, so Save stays live for the - * press that commits it. */ - let casesEditing = $state(false) /** The next free `_datasetN`, which is what a dataset is called until it is named. */ function nextDatasetIndex(): number { @@ -109,9 +101,9 @@ export function openDrawer(next: 'new' | 'edit') { mode = next pathError = '' - // The grid is rebuilt per open and reports afresh; a cell left open when the drawer closed - // would otherwise leave Save live over a set nothing has changed. - casesEditing = false + // Or the case added last time opens itself when the drawer is next opened on this dataset: + // ids survive `fromStoredCase`, so the id would still match a row. + focusCaseId = undefined // What was collected for a dataset that was never created belongs to that attempt. pendingScorers = [] workingCases = next === 'edit' ? cases.map((c) => fromStoredCase(c) as CaseDraft) : [] @@ -145,7 +137,6 @@ async function createDataset() { if (!workspace || !path || pathError) return - await casesGrid?.flush() creating = true const created = path const submittedSummary = summary || undefined @@ -164,9 +155,9 @@ creating = false return } - // The pane moves onto the created dataset before the drawer closes: closing reopens the Run - // dialog, which reads the pane's selection as it opens. A refresh that fails must still not - // read as a create that failed, or the retry hits "already exists". + // The pane moves onto the created dataset while the drawer is still open — the Run dialog + // stands underneath and follows that selection. A refresh that fails must still not read as + // a create that failed, or the retry hits "already exists". try { await onCreated(created) } catch (e) { @@ -193,8 +184,6 @@ * refuses the case edits with it instead of leaving them written under the old name. */ async function saveDataset() { if (!workspace || !datasetPath || !dataset || !path || pathError) return - // Before anything reads the list: a cell still open is an edit that must go in with it. - await casesGrid?.flush() // Read once, so every step of the save agrees on what was submitted: the fields are locked // while it runs, and the one the drawer navigates to afterwards must be the one written. // `summary` always travels: the server keeps the stored one when it is absent, so clearing @@ -233,8 +222,13 @@ saving = false } + /** The case the grid should open for typing: the one just added. */ + let focusCaseId = $state(undefined) + function addCase() { - workingCases = [...workingCases, { ...emptyCase(), id: randomUUID() }] + const id = randomUUID() + workingCases = [...workingCases, { ...emptyCase(), id }] + focusCaseId = id } function deleteCase(id: string) { @@ -263,9 +257,15 @@ ) }) let nothingToSave = $derived(metadataUnchanged && !casesChanged) + /** A dataset with no cases cannot be run — the server refuses it (`run.rs`, "has no case to + * run") — so there is no point creating one. Only creation is blocked: a saved dataset must + * stay editable down to its last case, or removing that case, or renaming while it is the only + * one left, would have no way to be saved. */ + let noCases = $derived(workingCases.length === 0) + let noCasesTitle = $derived(noCases ? 'Add at least one case first' : undefined) - onClosed?.()}> + (scorersWriting = w)} /> -
    + +
    Cases {workingCases.length} @@ -369,13 +372,13 @@ Add a case
    -
    +
    (casesEditing = v)} />
    @@ -397,7 +400,7 @@ unifiedSize="md" variant="accent" loading={saving} - disabled={writing || !path || !!pathError || (nothingToSave && !casesEditing)} + disabled={writing || !path || !!pathError || nothingToSave} onclick={saveDataset} > Save @@ -408,7 +411,8 @@ variant="accent" startIcon={{ icon: Plus }} loading={creating} - disabled={creating || !path || !!pathError} + disabled={creating || !path || !!pathError || noCases} + title={noCasesTitle} onclick={createDataset} > Create dataset diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte index d4d1714fd0..63fe366628 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte @@ -47,16 +47,6 @@ let hasDraft = $derived(editedConfig !== undefined) let dataset = $state(undefined) let hoveringDataset = $state(false) - /** Set while the dialog stands aside for the dataset drawer, holding the pane's dataset as it - * was on the way out: coming back onto a different one is the detour's answer, and adopted. */ - let steppedAside = $state<{ dataset: string | undefined } | undefined>(undefined) - - /** Hands the screen to the dataset drawer, to be given back when it closes. */ - function stepAside(go: () => void) { - steppedAside = { dataset: defaultDataset } - open = false - go() - } /** The agent's versions, for pinning one. Loaded when the dialog opens rather than held: a * version list goes stale the moment the agent is saved again. */ @@ -76,21 +66,26 @@ } } + /** Whether the dialog was already open on the previous run of the effect below, which reads it + * to tell an open from the pane's dataset moving underneath. */ + let wasOpen = false + $effect(() => { - if (!open) return + const isOpen = open + const pane = defaultDataset untrack(() => { - loadVersions() - const aside = steppedAside - steppedAside = undefined - if (aside) { - // Back from the drawer: a dataset created or edited there moves the pane onto it, and - // anything else leaves the field as it was left. - if (defaultDataset !== aside.dataset) dataset = defaultDataset + if (!isOpen) { + wasOpen = false return } - // Seeded on every open: the dataset last worked in, and the state of the agent there is - // most reason to measure. - dataset = defaultDataset + // Followed while the dialog stands, not only read at open: the dataset drawer opens over + // this dialog rather than in place of it, so creating, renaming or deleting a dataset + // there moves the pane's selection with the dialog still up. Nothing else moves it then. + dataset = pane + if (wasOpen) return + wasOpen = true + loadVersions() + // The state of the agent there is most reason to measure. choice = hasDraft ? 'draft' : 'deployed' }) }) @@ -184,7 +179,7 @@ unifiedSize="sm" variant="default" startIcon={{ icon: Plus }} - onclick={() => stepAside(onNewDataset)} + onclick={onNewDataset} > New dataset @@ -215,7 +210,7 @@ title="Edit this dataset" on:click={() => { close() - stepAside(() => onEditDataset(item.value ?? '')) + onEditDataset(item.value ?? '') }} /> {/snippet} @@ -228,7 +223,7 @@ btnClasses="w-full !h-auto !justify-start !rounded-none flex items-center gap-2 px-3 py-2 text-xs !font-normal text-secondary hover:bg-surface-hover" onClick={() => { close() - stepAside(onNewDataset) + onNewDataset() }} > @@ -247,7 +242,7 @@ startIcon={{ icon: Pencil }} iconOnly title="Edit this dataset" - on:click={() => stepAside(() => onEditDataset(dataset ?? ''))} + on:click={() => onEditDataset(dataset ?? '')} />
    {/if} diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte index 5d84ef88ce..b544781902 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -7,17 +7,21 @@ import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import TimeAgo from '$lib/components/TimeAgo.svelte' import { Button } from '$lib/components/common' - import { Bot, Code2, Loader2, Plus } from 'lucide-svelte' + import { Bot, ChevronRight, Code2, Loader2, Plus } from 'lucide-svelte' + import { overlayHostActive, topmostSurface } from '$lib/components/common/overlayHost.svelte' import type { EvalDataset, EvalExperiment, ExperimentScore } from '$lib/gen' import { datasetSummary, experimentName, formatScore, subjectLabel } from './evalUtils' let { experiments, datasets, + caseProgress, loaded, + active = false, deployedHash = undefined, currentVersion = undefined, onOpen, + onHighlight, onEditDataset, onNew }: { @@ -25,17 +29,87 @@ experiments: EvalExperiment[] /** Whether the list has been read: an empty table is a statement about the agent. */ loaded: boolean + /** Whether this list is the page on screen. The keyboard is only answered while it is: the + * run page keeps its own rows, and both would otherwise move on one press. */ + active?: boolean /** The workspace's datasets, for naming the one a run is of by what it is for. */ datasets: EvalDataset[] + /** How many cases each still-running run has finished, keyed by run id. A run the flow has + * not been read for yet is at none of them rather than absent: the count is on the row from + * the moment it appears, so it never arrives late and shifts the column. */ + caseProgress: Record /** What the agent hashes to as deployed, and the version it is on: they resolve a run of * edits that were later saved, so a run is labelled here as the run picker labels it. */ deployedHash?: string currentVersion?: number onOpen: (experiment: EvalExperiment) => void + /** The highlighted run, reported up so the surface can act on it — arrowing into the run + * page opens the run under the highlight rather than whichever was opened last. */ + onHighlight?: (id: string | undefined) => void onEditDataset: (dataset: string) => void onNew: () => void } = $props() + /** The highlighted run, by id. One state for both the pointer and the keyboard, as a melt menu + * does it: hovering a row moves the highlight to it, so the arrows carry on from wherever the + * pointer left off instead of running a second, invisible cursor of their own. It says where + * the highlight is, not what is chosen — a run is not opened until Enter. + * + * By id and not by index: the list is newest-first and the poll prepends to it, so an index + * would quietly come to mean a different run and Enter would open the wrong one. */ + let cursorId = $state(undefined) + let cursor = $derived( + cursorId === undefined ? -1 : experiments.findIndex((e) => e.id === cursorId) + ) + let body: HTMLTableSectionElement | undefined = $state() + + // A window listener answers keys aimed anywhere, so it has to ask two questions the DOM cannot: + // is my host the visible one — session preview tabs stay mounted when hidden — and is my surface + // still the one on top, rather than under a drawer or a dialog opened since. + const hostActive = overlayHostActive() + const onTop = topmostSurface() + const listening = () => hostActive() && onTop() + + $effect(() => { + onHighlight?.(cursorId) + }) + + // A highlight on a run that has since gone, and the highlight itself when the list is not the + // page on screen. + $effect(() => { + if (!active || (cursorId !== undefined && cursor < 0)) cursorId = undefined + }) + + function move(by: number) { + if (experiments.length === 0) return + const from = cursor < 0 ? (by > 0 ? -1 : experiments.length) : cursor + const at = Math.max(0, Math.min(experiments.length - 1, from + by)) + cursorId = experiments[at]?.id + // `nearest`, so arrowing through a long list scrolls by a row rather than jumping the table. + requestAnimationFrame(() => + body?.querySelectorAll('tr')[at]?.scrollIntoView({ block: 'nearest' }) + ) + } + + function onKeydown(event: KeyboardEvent) { + if (!active || !listening() || event.metaKey || event.ctrlKey || event.altKey) return + const el = event.target as HTMLElement | null + if (el?.closest?.('input, textarea, select, [contenteditable="true"], [role="listbox"]')) return + if (event.key === 'ArrowDown') { + event.preventDefault() + move(1) + } else if (event.key === 'ArrowUp') { + event.preventDefault() + move(-1) + } else if (event.key === 'Enter' && experiments[cursor]) { + // Enter belongs to whatever is focused if that thing does something with it. A highlighted + // row is not a reason to swallow the press on `New evaluation` or a row's dataset button. + if (el?.closest?.('button, a[href], [role="button"], summary')) return + event.preventDefault() + onOpen(experiments[cursor]) + } + } + /** The one number a column reports: a pass rate where it has a line to pass, the mean where it * does not. */ function headline(score: ExperimentScore): string | undefined { @@ -44,13 +118,16 @@ } + +
    - + + @@ -58,12 +135,20 @@ Dataset Cases Scores - When + When + - - {#each experiments as experiment (experiment.id)} - onOpen(experiment)}> + + {#each experiments as experiment, i (experiment.id)} + + onOpen(experiment)} + on:hover={(e) => e.detail && (cursorId = experiment.id)} + >
    @@ -99,7 +184,18 @@ - {experiment.case_count} + {#if experiment.running} + + + + {caseProgress[experiment.id] ?? 0}/{experiment.case_count} + + {:else} + {experiment.case_count} + {/if}
    @@ -117,8 +213,6 @@ {value} {:else if score.failed > 0} failed - {:else if experiment.running} - {:else} — {/if} @@ -126,33 +220,34 @@ {/each} {#if (experiment.scores ?? []).length === 0} - {#if experiment.running} - - - scoring - - {:else} - not scored - {/if} + + not scored {/if}
    - + + + + {/each} {#if experiments.length === 0 && !loaded}
    - {:else if experiments.length === 0} - - - - {#each scorers as scorer (scorer.id)} - - {/each} - - - - Case - Answer - {#each scorers as scorer, index (scorer.id)} - {@const mean = means.find((m) => m.scorer_id === scorer.id)} - {@const headline = columnHeadline(scorer, mean)} - - -
    - - {#if scorer.kind === 'agent'} - - {:else} - - {/if} - {scorerLabel(scorer)} - - - {#if headline} - - {headline.value} + +
    + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
    + + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} - {#if headline.delta && headline.direction !== 0} - 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} - > - {headline.delta} - - {/if} {/if} - -
    -
    - {/each} + {/if} + + + + {/each} +
    + + + {#if datasetLoading} + + + - - + {:else} {#each displayRows as row (row.case_id)} {@const status = statusOf(row.status)} {/each} - - - {/if} + {/if} + + - {#if selectedRow} - {@const openRow = selectedRow} - + + + {#if openRow}
    -
    - + +
    + {openRow.input?.user_message ?? caseLabel(openRow)} -
    - {#if openRow.job_id} - - Open the case job - - - {/if} -
    -
    + +
    {#if openRow.expected != undefined && openRow.expected !== ''} - +
    +
    + Expected +
    +
    + + {typeof openRow.expected === 'string' + ? openRow.expected + : JSON.stringify(openRow.expected, null, 2)} + +
    +
    {/if} {#if scorers.length > 0 && openRow.scores.length > 0} -
    -
    +{/snippet} { - if (await useDataset(path)) { - resumeRunDialog = true - datasetDrawer?.openDrawer('edit') - } - }} - onNewDataset={() => { - resumeRunDialog = true - datasetDrawer?.openDrawer('new') + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') }} + onNewDataset={() => datasetDrawer?.openDrawer('new')} /> { - if (!resumeRunDialog) return - resumeRunDialog = false - // On the dataset the drawer was just in: the dialog opens on the pane's own, which - // creating or editing one has already moved to it. - runDialogOpen = true - }} /> + + diff --git a/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte new file mode 100644 index 0000000000..a1a58a4ce5 --- /dev/null +++ b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte @@ -0,0 +1,219 @@ + + +
    +
    +
    + {#if context} + + {context} + + {/if} + {#if job} +

    + Requested by {job.created_by} · +

    + {/if} +
    + {STATUS_BADGE[status].label} +
    + + {#if typeof approvalInfo.description === 'string'} +

    {approvalInfo.description}

    + {:else if approvalInfo.description != undefined} + + {/if} + + {#if status === 'pending'} + {#if hasForm} + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} + {/if} + + {#if approvalInfo.can_approve} +
    + {#if approvalInfo.hide_cancel !== true} + + {:else} +
    + {/if} + +
    + {#if isSelfApprovalBypass} + + As an administrator, by approving or rejecting this request, you bypass the self-approval + interdiction. + + {/if} + {:else if approvalInfo.user_auth_required && !$userStore} +

    Sign in to review this request.

    + + {:else} +
    +

    You are not authorized to approve this request.

    + {#if isSelfApprovalRefused} +

    Self-approval is disabled for this step.

    + {/if} + {#if groupsRequired.length > 0} +

    + Only members of the following groups can approve: + {groupsRequired.join(', ')} +

    + {/if} +
    + {/if} + {:else} +
    + {#if status === 'approved'} + + {:else if status === 'rejected'} + + {:else} + + {/if} +
    + + {status === 'closed' ? 'This request is closed' : STATUS_BADGE[status].label} + + + {#if status === 'approved'} + Your approval was recorded. You can close this page. + {:else if status === 'rejected'} + Your rejection was recorded. You can close this page. + {:else} + The flow is no longer waiting for approval. + {/if} + +
    +
    + {/if} + + {#if !isLocked && ((status === 'pending' && approvalInfo.approvers.length > 0) || isWorkspaceMember)} +
    + + {#if status === 'pending' && approvalInfo.approvers.length > 0} + Already approved by {approvalInfo.approvers.map((a) => a.approver).join(', ')} + {/if} + + {#if isWorkspaceMember} + + {/if} +
    + {/if} +
    diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..f2337d699b 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -282,7 +282,8 @@ const scriptsV2: typeof legacyScripts = { ...legacyScripts.postgresql, code: ` SELECT table_name, column_name, udt_name, column_default, is_nullable, nsp.nspname AS table_schema FROM information_schema.columns -RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')` +RIGHT JOIN pg_namespace nsp ON table_schema = nsp.nspname WHERE nsp.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') +AND NOT starts_with(nsp.nspname, 'pg_') AND has_schema_privilege(nsp.oid, 'USAGE')` } } @@ -333,25 +334,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css deleted file mode 100644 index 1ac9f8b31c..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css +++ /dev/null @@ -1,26 +0,0 @@ -/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame - rather than inherit it. */ -.ag-theme-alpine .wm-multiline-cell-editor, -.ag-theme-alpine-dark .wm-multiline-cell-editor { - background-color: var(--ag-background-color); -} -.ag-theme-alpine .wm-multiline-cell-editor textarea, -.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { - display: block; - box-sizing: border-box; - /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row - it is replacing. `line-height` here is what it computes against. */ - padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); - border: 1px solid var(--ag-input-focus-border-color); - border-radius: 3px; - outline: none; - resize: none; - /* Past this it scrolls rather than growing. */ - max-height: 40vh; - overflow-y: auto; - background-color: var(--ag-background-color); - color: var(--ag-foreground-color); - font: inherit; - line-height: 20px; - white-space: pre-wrap; -} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts deleted file mode 100644 index 00975955c9..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' -// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule -// added to it is one the next copy of it drops. -import './multilineCellEditor.css' - -/** Kept in step with the `line-height` the stylesheet gives the textarea. */ -const LINE_HEIGHT = 20 - -/** - * A text cell editor that starts the height of the cell and grows as lines are added, for columns - * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. - * - * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so - * growing is only visible if the editor is allowed to paint outside it. - */ -export class MultilineCellEditor implements ICellEditorComp { - private eGui!: HTMLDivElement - private textarea!: HTMLTextAreaElement - private params!: ICellEditorParams - private wasEmpty = false - - init(params: ICellEditorParams) { - this.params = params - this.eGui = document.createElement('div') - this.eGui.className = 'wm-multiline-cell-editor' - - this.wasEmpty = params.value == undefined - - this.textarea = document.createElement('textarea') - this.textarea.rows = 1 - // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 - // and double-click keep it to be edited. - this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') - this.textarea.style.width = `${params.column.getActualWidth() - 2}px` - // Padded so one line fills the cell it replaces and a second costs a line rather than a row. - // From the row rather than from `--ag-row-height`, which is the theme's figure and not - // necessarily this grid's. - const rowHeight = params.node.rowHeight ?? 28 - const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) - this.textarea.style.paddingTop = `${padding}px` - this.textarea.style.paddingBottom = `${padding}px` - - this.textarea.addEventListener('input', () => this.resize()) - this.textarea.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a - // surface that closes on Escape, and leaving an edit is not asking to leave that. - e.preventDefault() - e.stopPropagation() - this.params.api.stopEditing(true) - return - } - if (e.key !== 'Enter' || e.isComposing) return - // Both branches keep the key from the grid, which ends the edit on Enter whether or not - // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter - // ends the edit here instead. - e.stopPropagation() - if (!e.shiftKey) { - e.preventDefault() - this.params.stopEditing() - } - }) - this.eGui.appendChild(this.textarea) - } - - private resize() { - this.textarea.style.height = 'auto' - this.textarea.style.height = `${this.textarea.scrollHeight}px` - } - - getGui() { - return this.eGui - } - - afterGuiAttached() { - this.resize() - this.textarea.focus() - // At the end rather than selected: a selection is a keystroke away from erasing the cell. - const end = this.textarea.value.length - this.textarea.setSelectionRange(end, end) - } - - getValue() { - // Nothing typed into a cell that held nothing is not an edit: returning '' here would write - // an empty string over a null, which the grid would see as a change and commit. - if (this.wasEmpty && this.textarea.value === '') return this.params.value - return this.textarea.value - } - - isPopup() { - return true - } - - getPopupPosition(): 'over' | 'under' { - return 'over' - } -} - -/** - * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as - * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit - * under, so the editor cannot keep Shift+Enter for itself on its own. - */ -export const multilineCellColDef: Pick = { - cellEditor: MultilineCellEditor, - suppressKeyboardEvent: (p) => - p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey -} diff --git a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte index aba0569228..7ec2336d3b 100644 --- a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte +++ b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte @@ -19,6 +19,7 @@ import { computeGlobalContext, eval_like } from './eval' import { deepEqual } from 'fast-equals' import { deepMergeWithPriority, isCodeInjection, readFieldsRecursively } from '$lib/utils' + import { escapeTemplateBackticks } from '$lib/utils/templateLiteral' import sum from 'hash-sum' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' @@ -271,7 +272,7 @@ if ((input.type === 'template' || input.type == 'templatev2') && isCodeInjection(input.eval)) { try { const r = await eval_like( - '`' + input.eval.replaceAll('`', '\\`') + '`', + '`' + escapeTemplateBackticks(input.eval) + '`', computeGlobalContext($worldStore, id, fullContext), $stateStore, $mode == 'dnd', diff --git a/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte b/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte index 9d1302233f..83976a3ecd 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppQuillEditor.svelte @@ -8,7 +8,7 @@ import InputValue from '../helpers/InputValue.svelte' import InitializeComponent from '../helpers/InitializeComponent.svelte' - let editor = $state() + let editor: HTMLElement | undefined = $state() let quill: any = $state() interface Props { id: string @@ -24,7 +24,7 @@ toolbarOptions = [ [{ header: 1 }, { header: 2 }, 'blockquote', 'link', 'image', 'video'], ['bold', 'italic', 'underline', 'strike'], - [{ list: 'ordered' }, { list: 'ordered' }], + [{ list: 'ordered' }, { list: 'bullet' }], [{ align: [] }], ['clean'] ] @@ -42,6 +42,7 @@ async function loadQuill() { const { default: Quill } = await import('quill') + if (!editor) return quill = new Quill(editor, { modules: { @@ -51,33 +52,73 @@ placeholder: placeholder }) - if (defaultValue) { - quill.root.innerHTML = defaultValue - } - quill.on('text-change', function (delta, oldDelta, source) { setOutput() }) + + if (defaultValue) { + setHtml(defaultValue) + } + } + + // Load HTML through Quill's clipboard converter rather than assigning `root.innerHTML`: + // Quill only keeps markup it has a blot for, and writes to the DOM bypass that mapping. + // Quill 2 renders every list as `
      `, so a `
        ` from stored content would be dropped + // outright instead of becoming a bullet list. + function setHtml(html: string | undefined) { + quill.setContents(quill.clipboard.convert({ html: html ?? '', text: '' })) } $componentControl[untrack(() => id)] = { setValue(nvalue: string) { if (quill) { - quill.root.innerHTML = nvalue + setHtml(nvalue) setOutput() } } } + // Quill 2 keeps every list in an `
          ` and marks the kind with `data-list` on each + // item (`bullet`, `ordered`, `checked`, `unchecked`), plus a `span.ql-ui` that only its + // stylesheet renders. Consumers of the output (an HTML component, stored content) render + // it outside that stylesheet, where that markup reads as a numbered list. Emit the + // standard shape instead: `
            `/`
              ` runs, checklists as `
                `, which + // is what the clipboard converter reads back on load (`matchList`). + function toStandardHtml(root: HTMLElement): string { + const clone = root.cloneNode(true) as HTMLElement + clone.querySelectorAll('span.ql-ui').forEach((s) => s.remove()) + clone.querySelectorAll('ol').forEach((ol) => { + const lists: HTMLElement[] = [] + for (const li of Array.from(ol.children)) { + const kind = li.getAttribute('data-list') + li.removeAttribute('data-list') + const tag = kind === 'ordered' ? 'ol' : 'ul' + const checked = kind === 'checked' ? 'true' : kind === 'unchecked' ? 'false' : null + const last = lists.at(-1) + if ( + last?.tagName.toLowerCase() !== tag || + last.getAttribute('data-checked') !== checked + ) { + const list = document.createElement(tag) + if (checked !== null) list.setAttribute('data-checked', checked) + lists.push(list) + } + lists.at(-1)!.appendChild(li) + } + ol.replaceWith(...lists) + }) + return clone.innerHTML + } + function setOutput() { if (quill) { - outputs?.result.set(quill.root.innerHTML) + outputs?.result.set(toStandardHtml(quill.root)) } } function handleDefault(defaultValue: string | undefined) { if (quill) { - quill.root.innerHTML = defaultValue + setHtml(defaultValue) setOutput() } } diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 8ef4ffbf95..073d6c3e7d 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -85,7 +85,8 @@ loadedFromDraft = false, othersDraftsCount = 0, onOpenOthersDrafts, - onRestore + onRestore, + onDeploy }: AppEditorProps = $props() migrateApp(untrack(() => app)) @@ -473,15 +474,6 @@ let appEditorHeader: AppEditorHeader | undefined = $state(undefined) - export function triggerTutorial() { - const urlParams = new URLSearchParams(window.location.search) - const tutorial = urlParams.get('tutorial') - - if (tutorial) { - appEditorHeader?.runTutorialById(tutorial) - } - } - let box: HTMLElement | undefined = $state(undefined) function parseScroll() { $yTop = box?.scrollTop ?? 0 @@ -902,6 +894,7 @@ {othersDraftsCount} {onOpenOthersDrafts} {onRestore} + {onDeploy} {policy} {fromHub} bind:this={appEditorHeader} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index a57dc87622..8a0f11c5a1 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -7,25 +7,13 @@ import { redo, undo } from '$lib/history.svelte' import { discardDraftAfterDeploy } from '$lib/userDraftToast' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' - import { - enterpriseLicense, - tutorialsToDo, - userStore, - userWorkspaces, - workspaceStore - } from '$lib/stores' + import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { isMac, type Item, userPathPrefix } from '$lib/utils' - import { resetAllTodos, skipAllTodos } from '$lib/tutorialUtils' - import { getTutorialIndex } from '$lib/tutorials/config' import { random_adj } from '$lib/components/random_positive_adjetive' import { AlignHorizontalSpaceAround, BellOff, - BookOpen, Bug, - CheckCheck, - CheckCircle, - Circle, DiffIcon, Expand, FileJson, @@ -33,7 +21,6 @@ FormInput, History, Laptop2, - RefreshCw, Save, Smartphone, FileClock, @@ -45,7 +32,7 @@ Zap, Globe } from 'lucide-svelte' - import { getContext, untrack } from 'svelte' + import { getContext, onDestroy, untrack } from 'svelte' import { orderedJsonStringify, type Value, replaceFalseWithUndefined } from '../../../utils' import type { App, AppEditorContext, AppViewerContext } from '../types' import { toStatic } from '../utils' @@ -61,7 +48,6 @@ import Awareness from '$lib/components/Awareness.svelte' import { secondaryMenuLeftStore, secondaryMenuRightStore } from './settingsPanel/secondaryMenu' import Dropdown from '$lib/components/DropdownV2.svelte' - import AppEditorTutorial from './AppEditorTutorial.svelte' import AppReportsDrawer from './AppReportsDrawer.svelte' import DebugPanel from './contextPanel/DebugPanel.svelte' @@ -124,6 +110,10 @@ // (not `on:restore` forwarding): forwarding a `createEventDispatcher` // event up through these runes-mode components silently drops it. onRestore?: (restoredApp: any) => void + // Fired after a successful deploy, which keeps this editor open: `version` is what + // the deploy wrote, for the next draft's fork base, and `head` what is deployed + // now, with its author and time. + onDeploy?: (e: { version?: number; head?: number; headBy?: string; headAt?: string }) => void } let { @@ -151,7 +141,8 @@ loadedFromDraft = false, othersDraftsCount = 0, onOpenOthersDrafts, - onRestore + onRestore, + onDeploy }: Props = $props() /** Mirror of the path the user is editing in the pen popover. Initialized @@ -343,13 +334,24 @@ } } - async function syncWithDeployed() { + /** The opening this editor claimed last. A path change remounts this editor while the + * drawer stays mounted, so its teardown hands that opening back rather than leaving + * the drawer on the app the user left. */ + let lastOpening: number | undefined = undefined + onDestroy(() => { + if (lastOpening != null) diffDrawer?.abandonOpening(lastOpening) + }) + + async function syncWithDeployed(opening?: number) { const deployedApp = await AppService.getAppByPath({ workspace: $workspaceStore!, path: $appPath!, withStarredInfo: true }) + // A superseded opening must not write these: the current one would then render + // against the older deployed value. + if (opening != null && !diffDrawer?.ownsOpening(opening)) return deployedBy = deployedApp.created_by // Strip off extra information @@ -365,7 +367,7 @@ async function updateApp(npath: string) { policy = await updatePolicy($app, policy) - await AppService.updateApp({ + const deployed = await AppService.updateApp({ workspace: $workspaceStore!, path: $appPath!, requestBody: { @@ -391,16 +393,26 @@ custom_path: customPath, labels: $state.snapshot(labels) } - const appHistory = await AppService.getAppHistoryByPath({ + const head = await AppService.getAppLatestVersion({ workspace: $workspaceStore!, path: npath }) - version = appHistory[0]?.version - // Re-pin the fork base to the just-deployed head: the editor stays open, so a + // `version` is what is deployed now, which is the deploy guard's fallback head; the + // deploy's own answer is the base, and the two differ when another landed beside it. + version = head?.version + // Re-pin the fork base to the version just written: the editor stays open, so a // follow-up deploy (or a new edit) would otherwise compare against the now- // superseded base and falsely warn. parent_version is in // DRAFT_COMPARE_IGNORED_FIELDS, so this write can't spawn a spurious draft. - if ($app) $app.parent_version = version + if ($app) $app.parent_version = deployed.version + // The route owns the pair the out-of-date prompt reads, and this editor stays open + // across the deploy, so hand both over rather than leaving it on the old ones. + onDeploy?.({ + version: deployed.version, + head: version, + headBy: head?.created_by, + headAt: head?.created_at + }) closeSaveDrawer() sendUserToast('App deployed successfully') @@ -444,6 +456,9 @@ } let onLatest = $state(true) + /** The last comparison could not read the head, so the confirmation it raises is + * caution and not an observed deploy. Cleared by the next reading comparison. */ + let headUnknown = $state(false) async function compareVersions() { // Compare the draft's pinned fork base (`$app.parent_version`) against the // current head when editing a draft, else the load-time head. Catches both a @@ -459,9 +474,14 @@ path: $appPath }) onLatest = appVersion?.version === undefined || base === appVersion?.version + headUnknown = false } catch (e) { console.error('Error comparing versions', e) - onLatest = true + // The head is what this compares against, so an unanswered read is not + // evidence of being current: confirm, as an unclaimable deploy does, and say + // that is why rather than claiming a version that was never seen. + onLatest = false + headUnknown = true } } @@ -638,23 +658,33 @@ if (!savedApp || newApp) { return } + // The fetch below is awaited, so a reopen (or a path change, which remounts + // this editor but not the drawer) while it runs must not have the older one + // land last. The drawer counts the openings for that reason. + const opening = diffDrawer?.beginOpening() + lastOpening = opening + if (opening == null) return // deployedValue should be syncronized when we open Diff - await syncWithDeployed() + await syncWithDeployed(opening) - diffDrawer?.openDrawer() - diffDrawer?.setDiff({ - mode: 'normal', - deployed: deployedValue ?? savedApp, - current: { - summary: $summary, - value: $app, - path: newEditedPath || savedApp.path, - policy, - custom_path: customPath, - labels - } - }) + if (!diffDrawer?.ownsOpening(opening)) return + diffDrawer.openDrawer(opening) + diffDrawer.setDiff( + { + mode: 'normal', + deployed: deployedValue ?? savedApp, + current: { + summary: $summary, + value: $app, + path: newEditedPath || savedApp.path, + policy, + custom_path: customPath, + labels + } + }, + opening + ) }, disabled: !savedApp || newApp }, @@ -679,49 +709,9 @@ action: () => { appExport?.open(toStatic($app, $staticExporter, $summary).app) } - }, - { - displayName: 'Tutorials', - icon: BookOpen, - separatorTop: true, - submenuItems: [ - { - displayName: 'Background runnables', - action: () => appEditorTutorial?.runTutorialById('backgroundrunnables'), - icon: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? Circle - : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) - ? undefined - : 'green' - }, - { - displayName: 'Connection', - action: () => appEditorTutorial?.runTutorialById('connection'), - icon: $tutorialsToDo.includes(getTutorialIndex('connection')) ? Circle : CheckCircle, - iconColor: $tutorialsToDo.includes(getTutorialIndex('connection')) ? undefined : 'green' - }, - { - displayName: 'Reset tutorials', - action: () => resetAllTodos(), - icon: RefreshCw, - separatorTop: true - }, - { - displayName: 'Skip tutorials', - action: () => skipAllTodos(), - icon: CheckCheck - } - ] } ]) as Item[] - let appEditorTutorial: AppEditorTutorial | undefined = $state(undefined) - - export function runTutorialById(id: string, options?: { skipStepsCount?: number }) { - appEditorTutorial?.runTutorialById(id, options) - } - let appReportingDrawerOpen = $state(false) export function openTroubleshootPanel() { @@ -775,6 +765,8 @@ {confirmCallback} bind:open {diffDrawer} + claimOpening={() => (lastOpening = diffDrawer?.beginOpening())} + {headUnknown} bind:deployedValue currentValue={{ summary: $summary, @@ -813,33 +805,41 @@ if (!savedApp || newApp) { return } + // The other entry point into the same drawer, so it takes an opening too. + const opening = diffDrawer?.beginOpening() + lastOpening = opening + if (opening == null) return // deployedValue should be syncronized when we open Diff - await syncWithDeployed() + await syncWithDeployed(opening) + if (!diffDrawer?.ownsOpening(opening)) return saveDrawerOpen = false - diffDrawer?.openDrawer() - diffDrawer?.setDiff({ - mode: 'normal', - deployed: deployedValue ?? savedApp, - current: { - summary: $summary, - value: $app, - path: newEditedPath || savedApp.path, - policy, - custom_path: customPath, - labels - }, - button: { - text: 'Looks good, deploy', - onClick: () => { - if (newApp) { - createApp(newEditedPath) - } else { - handleUpdateApp(newEditedPath) + diffDrawer.openDrawer(opening) + diffDrawer.setDiff( + { + mode: 'normal', + deployed: deployedValue ?? savedApp, + current: { + summary: $summary, + value: $app, + path: newEditedPath || savedApp.path, + policy, + custom_path: customPath, + labels + }, + button: { + text: 'Looks good, deploy', + onClick: () => { + if (newApp) { + createApp(newEditedPath) + } else { + handleUpdateApp(newEditedPath) + } } } - } - }) + }, + opening + ) }} >
                @@ -872,6 +872,7 @@ {setPublishState} appPath={$appPath} {onLatest} + {headUnknown} {savedApp} bind:summary={$summary} bind:customPath @@ -1090,15 +1091,7 @@
                {/if}
                -
                - - {#if $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) || $tutorialsToDo.includes(getTutorialIndex('connection'))} - - {/if} -
                - +
                {#if hasErrors} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index f673711159..4f27eb6b26 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -2,14 +2,17 @@ import { Alert } from '$lib/components/common' import Badge from '$lib/components/common/badge/Badge.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' + import { enterpriseLicense, userStore } from '$lib/stores' import { Loader2 } from 'lucide-svelte' import Tooltip from '$lib/components/Tooltip.svelte' import ClipboardPanel from '$lib/components/details/ClipboardPanel.svelte' import { untrack } from 'svelte' - import { AppService, SettingService } from '$lib/gen' + import { AppService, SettingService, WorkspaceService } from '$lib/gen' + import type { GuestUsage } from '$lib/gen' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import Path from '$lib/components/Path.svelte' import { computeSecretUrl } from './appDeploy.svelte' import { base } from '$lib/base' @@ -22,6 +25,15 @@ } from '$lib/components/OnBehalfOfSelector.svelte' import { canUserBypassRuleKind, protectionRulesState } from '$lib/workspaceProtectionRules.svelte' import { FRONTEND_SDK_SCOPES } from '$lib/components/raw_apps/sdkScopes' + import { logFeatureUsage } from '$lib/utils/featureUsage' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspaceStore = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) const WM_DEPLOYERS_GROUP = 'wm_deployers' @@ -31,6 +43,7 @@ appPath, customPath = $bindable(), onLatest, + headUnknown = false, savedApp, summary = $bindable(), deploymentMsg = $bindable(), @@ -50,6 +63,9 @@ appPath: string customPath: string | undefined onLatest: boolean + /** The comparison behind `onLatest` could not read the deployed head, so the warning + * is caution rather than an observed newer version. */ + headUnknown?: boolean savedApp: any summary: string deploymentMsg: string | undefined @@ -70,27 +86,74 @@ * (`/secret_of/...` 404s with no `app` row) and renders a placeholder * instead of the eternally-spinning link. */ newApp?: boolean - /** Workspace the app is deployed to — the session's acting workspace when - * embedded in a session preview, else the navigation `$workspaceStore`. - * The secret-URL / custom-path / folder / on-behalf-of lookups must target - * it, not `$workspaceStore` (which stays on the nav workspace in a session). */ + /** Workspace the app is deployed to. The secret-URL / custom-path / folder / + * on-behalf-of lookups target it; defaults to the operating workspace. */ operatingWorkspace?: string } = $props() - const opWs = $derived(operatingWorkspace ?? $workspaceStore) + const opWs = $derived(operatingWorkspace ?? $operatingWorkspaceStore) - let isDeployer = $derived($userStore?.groups?.includes(WM_DEPLOYERS_GROUP) ?? false) + let isDeployer = $derived(actingUser?.groups?.includes(WM_DEPLOYERS_GROUP) ?? false) // Admins always pass the backend check. For everyone else, fail closed // while the workspace protection rules are still loading so the toggle // is never briefly enabled for a user the rules will end up restricting. let rulesetsLoaded = $derived(protectionRulesState.rulesets !== undefined) let canSetAnonymous = $derived( - !!$userStore?.is_admin || - !!$userStore?.is_super_admin || + !!actingUser?.is_admin || + !!actingUser?.is_super_admin || (rulesetsLoaded && canUserBypassRuleKind('RestrictAnonymousAppDeployment', $userStore ?? undefined)) ) - let canPreserve = $derived(!!$userStore?.is_admin || !!$userStore?.is_super_admin || isDeployer) + let canSetGuest = $derived( + !!actingUser?.is_admin || + !!actingUser?.is_super_admin || + (rulesetsLoaded && + canUserBypassRuleKind('RestrictGuestAppDeployment', $userStore ?? undefined)) + ) + // The three rungs of the access control, widest last. `viewer` is a fourth + // execution mode that this control never sets (it runs components as the viewer, + // which a guest cannot be), so an app in it shows as members-only here. + let accessMode = $derived( + policy.execution_mode == 'anonymous' + ? 'anonymous' + : policy.execution_mode == 'guest' + ? 'guest' + : 'publisher' + ) + // Undefined until loaded. An app can be set to `guest` while the workspace has + // guests off, in which case the mode is stored but inert -- say so rather than + // letting the publisher believe the app is open. + let guestAccessEnabled: boolean | undefined = $state(undefined) + let guestUsage: GuestUsage | undefined = $state(undefined) + // Whether the deployment can have guests at all; off, the mode is not on offer. The + // backend decides; the hostname stands in until it has answered, the shared cloud + // being the only deployment where guests are unavailable. + let guestsAvailable = $derived.by(() => guestUsage?.available ?? !isCloudHosted()) + + $effect(() => { + const ws = opWs + if (ws === undefined) return + untrack(() => { + WorkspaceService.getPublicSettings({ workspace: ws }) + .then((s) => (guestAccessEnabled = s.guest_access_enabled)) + .catch(() => (guestAccessEnabled = undefined)) + WorkspaceService.getGuestUsage({ workspace: ws }) + .then((u) => (guestUsage = u)) + .catch(() => (guestUsage = undefined)) + }) + }) + + function onAccessModeChange(mode: string | undefined) { + if (mode === undefined || mode === accessMode) return + policy.execution_mode = mode + // Same as sandbox: a not-yet-deployed app has no row to PATCH, so + // `setPublishState` would 404. The mode is carried by the first deploy's + // policy; persist incrementally only once the app exists. + if (savedApp && !newApp) { + setPublishState() + } + } + let canPreserve = $derived(!!actingUser?.is_admin || !!actingUser?.is_super_admin || isDeployer) let savedOnBehalfOfEmail = $derived(savedApp?.policy?.on_behalf_of_email) let savedOnBehalfOf = $derived(savedApp?.policy?.on_behalf_of) let onBehalfOfChoice: OnBehalfOfChoice = $state(undefined) @@ -128,6 +191,10 @@ }${customPath}` ) + // The app URL a guest JWT rides on: append `guest.` and the viewer authenticates the + // token as a seatless guest. Uses the custom URL when set, else the public secret URL. + let guestJwtBase = $derived(customPath !== undefined ? fullCustomUrl : secretUrlHref) + // When embedding a raw app in an iframe inside another Windmill app (or any // cross-origin-isolated page), the embedded document must set COEP. The // `wm_coep` flag opts the public app into the cross-origin isolation headers. @@ -182,8 +249,15 @@ {#if !onLatest} - - By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff. + + {headUnknown + ? "Deploying may overwrite changes made by other users. Press 'Deploy' to see diff." + : "By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff."}
                {/if} @@ -300,6 +374,12 @@ checked={policy.sandbox == true} on:change={(e) => { policy.sandbox = e.detail || undefined + // Counted where the toggle is flipped rather than where the policy is + // persisted: a not-yet-deployed app only mutates it locally, and skipping + // those would read as unused in the case where it is picked up front. + logFeatureUsage('app_sandbox', 'toggled', { + key: `${rawApp ? 'raw' : 'low_code'}:${e.detail ? 'on' : 'off'}` + }) // Frontend API access exists only for a sandboxed app, so turning // isolation off drops the declared scopes with it rather than leaving // them set but inert. @@ -389,34 +469,83 @@ {/if} {#if !hideSecretUrl} -

                Public URL

                +

                Access

                - {#if rulesetsLoaded && !canSetAnonymous} + {#if rulesetsLoaded && !canSetAnonymous && policy.execution_mode != 'anonymous'} - Making this app publicly accessible without login is restricted to workspace admins and - bypass users by a workspace protection rule + Opening this app to anyone with the link is restricted to workspace admins and bypass users + by a workspace protection rule + +
                + {/if} + {#if rulesetsLoaded && !canSetGuest && policy.execution_mode != 'guest' && guestsAvailable} + + Opening this app to guests is restricted to workspace admins and bypass users by a workspace + protection rule
                {/if}
                - { - policy.execution_mode = e.detail ? 'anonymous' : 'publisher' - // Same as sandbox: a not-yet-deployed app has no row to PATCH, so - // `setPublishState` would 404. The mode is carried by the first - // deploy's policy; persist incrementally only once the app exists. - if (savedApp && !newApp) { - setPublishState() - } - }} - disabled={!savedApp || (!canSetAnonymous && policy.execution_mode != 'anonymous')} - /> + onAccessModeChange(e.detail)} + disabled={!savedApp} + > + {#snippet children({ item })} + + + + {/snippet} + +
                +
                + {#if policy.execution_mode == 'anonymous'} + Anyone holding the secret URL below can open this app without signing in. + {:else if policy.execution_mode == 'guest'} + {#if !guestsAvailable} + Guests are not available on Windmill Cloud, so this app still admits members only. They + require a self-hosted instance or a dedicated Windmill Cloud deployment. + {:else if guestUsage && !guestUsage.instance_enabled} + A superadmin has turned guests off for this instance, so this app still admits members + only. + {:else if guestAccessEnabled === undefined} + Checking whether this workspace allows guests… + {:else if guestAccessEnabled === false} + Guests are turned off for this workspace, so this app still admits members only. A + workspace admin can turn them on in the workspace settings. + {:else} + Anyone your identity provider authenticates can open this app without a Windmill account. + They join no workspace. Members of this workspace can open it too. + {#if guestUsage} + {guestUsage.guest_count} of {guestUsage.free_allowance} free guests used across this instance + in the last {guestUsage.window_days} days; beyond that, {guestUsage.metered + ? 'every four guests count as one seat' + : 'new guests are refused until the count drops'}. + {/if} + {/if} + {:else} + Only workspace members with read access on this app can open it. + {/if}
                {#if !savedApp || newApp} @@ -451,8 +580,40 @@ {/if}
                + {#if embedMode && policy.execution_mode == 'guest' && guestAccessEnabled && guestJwtBase && guestsAvailable} +
                +
                + Embed for your own authenticated users (guest JWT) +
                +
                + To open this app for a user your own product already authenticates, mint a short-lived JWT + in your backend and append it to the app URL as guest.<jwt>. Each token + is its own seatless guest, confined to this app — no shared secret and no Windmill + account, unlike the plain secret URL above. +
                +
                + Windmill verifies the token against the workspace's guest JWT key (Workspace settings → + Guests) — a PEM public key or a JWKS URL{#if !isCloudHosted()}, or the instance's + configured issuer (JWT_EXT_JWKS_URL) when no workspace key is set{/if}. Set + the public half there; in your backend, sign each token with the matching + private key using RS256/384/512, PS256/384/512 or ES256/384 (symmetric HS* is + refused), carrying email, workspace_id = {opWs}, + app_path = {appPath} and exp (at most 24h ahead). +
                + +
                + Replace YOUR_GUEST_JWT with the token your backend signs per user. Past the instance's + free guest allowance a new guest email is refused (see the count above); guests already seen + in the window keep working. +
                +
                + {/if} +
                - {#if !($userStore?.is_admin || $userStore?.is_super_admin)} + {#if !(actingUser?.is_admin || actingUser?.is_super_admin)} Custom path can only be set by workspace admins @@ -473,7 +634,7 @@ options={{ right: 'Use a custom URL' }} - disabled={!$enterpriseLicense || !($userStore?.is_admin || $userStore?.is_super_admin)} + disabled={!$enterpriseLicense || !(actingUser?.is_admin || actingUser?.is_super_admin)} /> {#if customPath !== undefined} @@ -481,7 +642,7 @@
                Custom path
                - import AppTutorials from '../../AppTutorials.svelte' - import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' - - let appTutorials: AppTutorials | undefined = $state(undefined) - let targetTutorial: string | undefined = $state(undefined) - - export function runTutorialById(id: string, options?: { skipStepsCount?: number }) { - appTutorials?.runTutorialById(id, options) - } - - -) => { - targetTutorial = event.detail.detail - }} -/> - - { - targetTutorial = undefined - }} - on:confirmed={async () => { - window.open(`/apps/add?tutorial=${targetTutorial}`, '_blank') - }} -> -
                - This tutorial can only be run on a new app. -
                -
                diff --git a/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte b/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte index 39ecb51715..98d380bcb3 100644 --- a/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte +++ b/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte @@ -7,7 +7,6 @@ import JobArgs from '$lib/components/JobArgs.svelte' import LogViewer from '$lib/components/LogViewer.svelte' - import { workspaceStore } from '$lib/stores' import { BellOff, Loader2, RefreshCw } from 'lucide-svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' import { classNames, truncateRev, isFlowPreview } from '../../../utils' @@ -21,6 +20,9 @@ import type { Job } from '$lib/gen' import type { JobById } from '../types' import { createEventDispatcher, untrack } from 'svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { open?: boolean @@ -163,7 +165,7 @@
                @@ -192,7 +194,7 @@
                @@ -219,7 +221,7 @@ {#if job != undefined && 'result' in job && job?.result != undefined}
                diff --git a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte index 4ae67dbb0e..63bc17a8eb 100644 --- a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte +++ b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte @@ -9,17 +9,19 @@ cleanValueProperties, displayDate, emptyString, - orderedYamlStringify, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AppService, type AppWithLastVersion, type AppHistory } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { Skeleton } from '$lib/components/common' import Button from '$lib/components/common/button/Button.svelte' import { createEventDispatcher, untrack } from 'svelte' import { Pencil, ArrowRight, X, Loader2 } from 'lucide-svelte' import Select from '$lib/components/select/Select.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { appPath: string | undefined @@ -48,7 +50,7 @@ return cached } - const app = await AppService.getAppByVersion({ workspace: $workspaceStore!, id: version }) + const app = await AppService.getAppByVersion({ workspace: $operatingWorkspace!, id: version }) versionCache[version] = app return app } @@ -60,7 +62,7 @@ loading = true versions = await AppService.getAppHistoryByPath({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path: appPath }) loading = false @@ -90,7 +92,7 @@ return } await AppService.updateAppHistory({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, id: appId, version: appVersion, requestBody: { @@ -107,7 +109,9 @@ } function toVersionLabel(version: AppHistory): string { - return emptyString(version.deployment_msg) ? `Version ${version.version}` : version.deployment_msg! + return emptyString(version.deployment_msg) + ? `Version ${version.version}` + : version.deployment_msg! } let availableVersions = $derived( diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index d5839c6766..99ea07c8fd 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -22,19 +22,22 @@ let { workspace, - path, - editHref + path }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string } = $props() let app: any = $state(undefined) let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -103,11 +106,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/apps/editor/PublicApp.svelte b/frontend/src/lib/components/apps/editor/PublicApp.svelte index c0cf693e0f..6de8f9df0c 100644 --- a/frontend/src/lib/components/apps/editor/PublicApp.svelte +++ b/frontend/src/lib/components/apps/editor/PublicApp.svelte @@ -1,14 +1,14 @@ @@ -107,18 +111,6 @@ >Powered by   Windmill
                - - {#snippet userInfo(child)} -
                {child}
                - {/snippet} - -
                {#if $userStore} - {@render userInfo($userStore.username)} - {:else if globalUser} - {@render userInfo(globalUser.email)} - {:else}{/if} -
                {/if} {#if notExists} @@ -128,17 +120,31 @@
                {:else if noPermission} -
                This app requires read access
                -
                - {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} - You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of - {:else}You must be logged in and have read access to this app{/if}
                + {#if guestAppPath && !$userStore} +
                Sign in to open this app
                +
                + You do not need a Windmill account. Signing in lets you open this app and nothing else. +
                + {:else} +
                + This app requires read access +
                +
                + {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} + You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of + {:else}You must be logged in and have read access to this app{/if}
                + {/if}
                {#if !jwtError} - + {/if}
                {:else if app} @@ -170,27 +176,35 @@ )} style={app?.value?.['css']?.['app']?.['viewer']?.style} > - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> + {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
                + Reload the page to try again. +
                + {/await}
                {/if} {/key} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index 3ba1bac617..e994767488 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -23,15 +23,17 @@ * bearer token (no cookie); the raw wrapper document always carries `CSP: sandbox`. */ import { BROWSER } from 'esm-env' - import { OpenAPI } from '$lib/gen' + import { OpenAPI, UserService } from '$lib/gen' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import { Alert, Skeleton } from '$lib/components/common' + import Alert from '$lib/components/common/alert/Alert.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import { base } from '$app/paths' import { goto } from '$app/navigation' import Login from '$lib/components/Login.svelte' import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' + import { loadAppPreview } from './loadAppPreview' import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' @@ -49,7 +51,9 @@ fetchEmbedToken, onViewerReady, viewer, - viewerUrl + viewerUrl, + guestAppPath = undefined, + guestEntry = 'none' }: { /** Embedder-side: validate access + mint the scoped token. Throws with a * `.status` of 401 (login required) or 404 (not found). Pass @@ -68,6 +72,16 @@ * (`/apps/get`, auth-gated, with chrome) differs from the cookieless, * chrome-less viewer route (`/app_embed`). */ viewerUrl?: string + /** `/` when this app is open to guests. The embedder's + * login gate fires before the page's own load, so the page must resolve this + * up front and pass it down — otherwise a signed-out visitor is offered an + * ordinary sign-in that creates an account and still cannot open the app. */ + guestAppPath?: string | undefined + /** Whether the app admits guests, as far as the page has found out. The sign-in + * card waits while `pending` (a configured auto-login would otherwise start an + * ordinary sign-in) and refuses to offer one on `error` — a transient fault must + * not become an account and a seat. Nothing else waits on it. */ + guestEntry?: 'pending' | 'none' | 'guest' | 'error' } = $props() const EMBED_PARAM = 'wm_embed' @@ -194,7 +208,48 @@ } // ---------------------------- embedder mode ---------------------------- - let status: 'loading' | 'ready' | 'noPermission' | 'notExists' | 'sdkPrompt' = $state('loading') + type FrameStatus = 'loading' | 'ready' | 'noPermission' | 'notExists' | 'sdkPrompt' + let status = $state('loading') + /** Whether the visitor holds an account session, probed whenever the app denies + * them. An account this app still refuses is not something signing in again can + * fix — an identity with an account is never given a guest session — so the card + * gives way to an explanation. */ + let accountSession = $state<'unknown' | 'none' | 'held'>('unknown') + /** What the sign-in refused with, shown above the card until the next attempt. */ + let signInError: string | undefined = $state(undefined) + let deniedStatus: number | undefined = $state(undefined) + /** The sign-in card belongs on a 401, and on a 403 unless discovery has settled + * that the app is not open to guests: a 403 on a guest app is a session for another + * app of the workspace, which a fresh sign-in replaces. True while discovery is + * pending, so the skeleton shows rather than a flash of "Not found". */ + let offerSignIn = $derived( + status === 'noPermission' || + (status === 'notExists' && deniedStatus === 403 && guestEntry !== 'none') + ) + let signInDidNotHelp = $derived(offerSignIn && accountSession === 'held') + $effect(() => { + if (offerSignIn && accountSession === 'unknown') { + // Workspace-less, so it answers for an account and never for a guest + // (pinned to its workspace) or for nobody. + UserService.getCurrentEmail() + .then(() => (accountSession = 'held')) + .catch(() => (accountSession = 'none')) + } + }) + + // The stale guest session must be gone before the card mounts: it still + // authenticates here, so the popup's success poll would see it and complete the + // sign-in before the new session lands. The card waits on `staleGuestCleared`; a + // failed logout fails closed rather than offering a sign-in that cannot complete. + let staleGuestCleared = $state(false) + let staleGuestLogoutFailed = $state(false) + $effect(() => { + if (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared) { + UserService.logout() + .then(() => (staleGuestCleared = true)) + .catch(() => (staleGuestLogoutFailed = true)) + } + }) let embedToken: string | null = $state(null) let iframeEl: HTMLIFrameElement | undefined = $state(undefined) @@ -293,6 +348,10 @@ } finishReady() } catch (e: any) { + // 401: no session. 403 on an app that admits guests: a guest session for a + // different app of this workspace, which a fresh sign-in replaces. Either + // way the sign-in card is the answer; anything else is not found. + deniedStatus = e?.status status = e?.status === 401 ? 'noPermission' : 'notExists' } } @@ -363,7 +422,8 @@ if (unsandboxed || isRaw) { // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed - // raw app. + // raw app. A low-code app's runtime downloads alongside the app payload. + if (!isRaw) loadAppPreview().catch(() => {}) onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -472,6 +532,8 @@ onMount(() => { if (isViewer) { + // Only a sandboxed low-code app is ever framed as a viewer. + loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() // Announce readiness so the embedder sends us the token. @@ -514,7 +576,7 @@ {/if} {:else if status === 'loading'} -{:else if status === 'notExists'} +{:else if status === 'notExists' && !offerSignIn}
                There was an error loading the app, is the url correct? @@ -529,17 +591,58 @@ onContinue={onSdkConsentContinue} onDecline={onSdkConsentDecline} /> -{:else if status === 'noPermission'} +{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} + +{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} +
                + + The app could not be reached to find out who may open it. Reload to try again. + +
                +{:else if offerSignIn} -
                This app requires read access
                -
                - initEmbedder()} - popup - rd={page.url.pathname + page.url.search + page.url.hash} - /> -
                + {#if signInDidNotHelp} + +
                + You are signed in, but this app is not open to you +
                +
                + It is open to the people it was shared with{guestAppPath + ? ', and to guests who have no Windmill account' + : ''}. Ask the person who shared it to give your account access. +
                + {:else} + {#if guestAppPath} +
                Sign in to open this app
                +
                + You do not need a Windmill account. Signing in lets you open this app and nothing else. +
                + {:else} +
                + This app requires read access +
                + {/if} + {#if signInError} +
                + {signInError} +
                + {/if} +
                + { + signInError = undefined + accountSession = 'unknown' + initEmbedder() + }} + onLoginError={(message) => (signInError = message)} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> +
                + {/if} {:else if unsandboxed} + {#if selectionDbtHasColumns} +
                + +
                + {/if} + + {#if selectionDbt.raw_code} +
                + +
                + {/if}
                {:else}
                diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index 815db36b18..e719bfe2bd 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -19,12 +19,15 @@ XCircle } from 'lucide-svelte' import type { ScriptLang } from '$lib/gen' - import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { enterpriseLicense } from '$lib/stores' import { sendUserToast } from '$lib/utils' import { PIPELINE_LANGUAGES } from './pipelineLanguages' import type { PipelineOutputKind } from './pipelineTemplates' import type { DbtAssetProvenance } from './types' import DbtIcon from '$lib/components/icons/DbtIcon.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() // Shape used for both the data prop and the run callback. Drafts carry // `content` / `language` so the page-level run handler can dispatch to @@ -118,7 +121,7 @@ async function runProducers(e: MouseEvent) { e.stopPropagation() - if (!$workspaceStore || running || !data.onRunProducer) return + if (!$operatingWorkspace || running || !data.onRunProducer) return if (scriptProducers.length === 0) return running = true const handler = data.onRunProducer @@ -170,14 +173,14 @@ } const cols = Object.entries(d.columns ?? {}) if (cols.length) { - lines.push( - `columns: ${cols.map(([c, desc]) => (desc ? `${c} (${desc})` : c)).join(', ')}` - ) + lines.push(`columns: ${cols.map(([c, desc]) => (desc ? `${c} (${desc})` : c)).join(', ')}`) } if (d.freshness) { const f = d.freshness as Record const window = (k: string) => - f[k]?.count != null ? `${k.replace('_after', '')} after ${f[k].count}${f[k].period?.[0] ?? ''}` : '' + f[k]?.count != null + ? `${k.replace('_after', '')} after ${f[k].count}${f[k].period?.[0] ?? ''}` + : '' const windows = ['warn_after', 'error_after'].map(window).filter(Boolean) if (windows.length) lines.push(`freshness: ${windows.join(', ')}`) } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte index f43259b561..7cee1deb76 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte @@ -8,7 +8,6 @@ // seconds so the user sees status transitions without refreshing — // JobLoader handles streaming for the *selected* job. import { JobService, type Job } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { onDestroy, untrack } from 'svelte' import { displayDate } from '$lib/utils' import { CheckCircle2, Clock, History, Loader2, XCircle, Ban } from 'lucide-svelte' @@ -21,6 +20,9 @@ import { Popover } from '$lib/components/meltComponents' import { twMerge } from 'tailwind-merge' import DispatchEventsButton from '$lib/components/runs/DispatchEventsButton.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Producers of this asset. Each contributes its own listExtendedJobs @@ -88,7 +90,7 @@ // Run only when the *content* of the producer set changes (string // key) or when the parent explicitly bumps `refreshKey` after // dispatching a new run. The producerKey debounce alone isn't - // enough: refresh() synchronously reads $workspaceStore and + // enough: refresh() synchronously reads $operatingWorkspace and // runnableProducers before the first await, and Svelte 5 records // those as deps of the surrounding $effect — which would then // re-fire whenever the parent re-derives `producers` (i.e. on @@ -174,11 +176,11 @@ async function refresh(): Promise { if (refreshInFlight) return refreshInFlight - if (!$workspaceStore || runnableProducers.length === 0) { + if (!$operatingWorkspace || runnableProducers.length === 0) { jobs = [] return } - const ws = $workspaceStore + const ws = $operatingWorkspace // Capture the producer paths *now* — using runnableProducers // directly inside the await would re-read after the array // identity churned, defeating the in-flight guard. @@ -286,12 +288,12 @@ Open ↗ - {#if $workspaceStore} + {#if $operatingWorkspace} {/if} @@ -397,7 +399,7 @@
                diff --git a/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte new file mode 100644 index 0000000000..28afa5904e --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte @@ -0,0 +1,73 @@ + + + +{#snippet failure()} +
                + Part of this column lineage could not be loaded, so the trace may be incomplete. +
                +{/snippet} + +{#if loading && nodes.length === 0} +
                + + Loading column lineage +
                +{:else if nodes.length === 0} + {#if failed} +
                {@render failure()}
                + {/if} +{:else if graph} +
                + + {#if failed} + {@render failure()} + {/if} + {#if truncated} +
                + Showing the part of the trace nearest this relation. The lineage reaches further than one + view can draw. +
                + {/if} +
                +{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte index 1b86fa6853..5f5dc98b60 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte @@ -9,7 +9,6 @@ // just be noise. import DBTable from '$lib/components/DBTable.svelte' import { resource } from 'runed' - import { workspaceStore } from '$lib/stores' import { loadAllTablesMetaData } from '$lib/components/apps/components/display/dbtable/metadata' import { dbTableOpsWithPreviewScripts } from '$lib/components/dbOps' import { WorkspaceService } from '$lib/gen' @@ -19,6 +18,9 @@ import Button from '$lib/components/common/button/Button.svelte' import { base } from '$lib/base' import { twMerge } from 'tailwind-merge' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Asset path as parsed from `datatable:///
    user/group
    {owner_name} - {#if can_write && !restricted} -
    - { - const role = e.detail - // const wasInFolder = (folder?.owners ?? []).includes(folder) - // const inAcl = ( - // folder?.extra_perms ? Object.keys(folder?.extra_perms) : [] - // ).includes(folder) - if (role == 'admin') { - await FolderService.addOwnerToFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { - owner: owner_name - } - }) - } else if (role == 'writer') { - await FolderService.removeOwnerToFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { - owner: owner_name, - write: true - } - }) - } else if (role == 'viewer') { - await FolderService.removeOwnerToFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { - owner: owner_name, - write: false - } - }) - } - loadFolder() - }} - > - {#snippet children({ item })} - + +
    - {#if (can_write && owner_name != 'u/' + $userStore?.username) || $userStore?.is_admin} + + {/snippet} + + + {:else} + {perm.role} + {/if} + + +
    + + {#if ownerKindOf(perm.owner_name) === 'group' && !offOperating} +
    path_glob Glob relative to f/{name}/permissioned as
    - - -
    + +
    -
    -
    -
    user
    {member_name} - {#if can_write} +
    - {#if can_write} -
    user
    {email}
    ownermember
    +
    +
    No runs yet diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte index 68287b0fbc..d00e52d77e 100644 --- a/frontend/src/lib/components/aiEvals/EvalScorers.svelte +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -196,26 +196,31 @@
    -
    +
    {#if scorers.length === 0} -
    +
    A scorer reads one run and returns a number. Every run of this dataset is measured by all of them, which is what makes two runs comparable.
    {:else} -
    + +
    {#each scorers as scorer (scorer.id)} -
    +
    {#if scorer.kind === 'agent'} - + {:else} - + {/if}
    - + {scorerLabel(scorer)} - {scorer.path} + {scorer.path}
    {#if scorer.pass_if != undefined} diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte index d9e5b16fe4..db765607e5 100644 --- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -9,9 +9,11 @@ import Label from '$lib/components/Label.svelte' import Popover from '$lib/components/Popover.svelte' import { Splitpanes, Pane } from 'svelte-splitpanes' + import AnimatedPane from '$lib/components/splitPanes/AnimatedPane.svelte' import { type AgentDraft, AiEvalsService, + JobService, type EvalCase, type EvalDataset, type EvalExperiment, @@ -20,7 +22,6 @@ type Scorer, type ScorerMean } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { onDestroy, onMount, untrack } from 'svelte' import { @@ -35,9 +36,11 @@ Code2, ExternalLink } from 'lucide-svelte' + import PagedContent from '$lib/components/common/modal/PagedContent.svelte' import EvalDatasetDrawer from './EvalDatasetDrawer.svelte' import EvalRunsList from './EvalRunsList.svelte' import EvalRunDialog from './EvalRunDialog.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import { caseLabel, @@ -48,15 +51,23 @@ subjectLabel, type EvalsLocation } from './evalUtils' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() /** A dataset is capped at this many cases, so one page holds the whole set. */ const CASE_PAGE_SIZE = 1000 + /** The id the run's flow gives the loop over its cases (`CASES_NODE_ID` in `ai_evals/run.rs`). + * Looked up by id rather than by position: the flow has a step after the loop too. */ + const CASES_MODULE_ID = 'cases' + let { agentPath, opWorkspace = undefined, editedConfig = undefined, - location = $bindable() + location = $bindable(), + active = true }: { /** The agent under test. A dataset and its runs belong to an agent. */ agentPath: string @@ -69,9 +80,13 @@ /** The level the pane is on and the way out of it, reported up so the surface holding it * can put both in its header. Undefined at the root, which that surface already names. */ location?: EvalsLocation + /** False while the pane is parked off screen by a surface that keeps it mounted. Its own + * pages answer the arrow keys at `window`, which a parked instance would take from + * whatever is actually on screen. */ + active?: boolean } = $props() - let ws = $derived(opWorkspace ?? $workspaceStore) + let ws = $derived(opWorkspace ?? $operatingWorkspace) let datasets = $state([]) let dataset = $state(undefined) let selectedDataset = $state(undefined) @@ -91,12 +106,56 @@ /** What the agent hashes to as deployed: a run of edits carrying it ran what was then saved. */ let deployedHash = $state(undefined) let running = $state(false) - let scorers = $derived(dataset?.scorers ?? []) + /** The run on screen belongs to a dataset still being read. Until it arrives the rows and the + * scorer columns would both be built from the *previous* dataset, so both are held back. */ + let datasetLoading = $state(false) + let scorers = $derived(datasetLoading ? [] : (dataset?.scorers ?? [])) let selectedCaseId = $state(undefined) let datasetDrawer: EvalDatasetDrawer | undefined = $state() let runDialogOpen = $state(false) - let resumeRunDialog = $state(false) + /** The run the list has highlighted, so arrowing into the run page opens that one. Without it + * the arrow could only fall back to whichever run was opened last, which on a dialog just + * opened is none at all. */ + let highlightedRunId = $state(undefined) + + /** How many cases each still-running run has finished, keyed by run id. Read from the flow + * executing the run: the list carries the case total, and counting the finished ones there + * would be a per-case query for every run listed. The flow already records it — one slot per + * case in `flow_jobs_success`, null until that case's iteration is over. */ + let caseProgress = $state>({}) + + async function readCaseProgress() { + const workspace = ws + const live = experiments.filter((e) => e.running) + if (!workspace || live.length === 0) { + if (Object.keys(caseProgress).length > 0) caseProgress = {} + return + } + const read = await Promise.all( + live.map(async (e) => { + try { + const update = await JobService.getJobUpdates({ + workspace, + id: e.run_job_id, + running: true, + noLogs: true + }) + const cases = update.flow_status?.modules?.find((m) => m.id === CASES_MODULE_ID) + if (!cases) return undefined + return [ + e.id, + (cases.flow_jobs_success ?? []).filter((s) => s != undefined).length + ] as const + } catch { + // Left out of the map, so the row reads `0/total` until a later poll answers. A + // flow that cannot be read is already the list's problem to report, not this one's. + return undefined + } + }) + ) + caseProgress = Object.fromEntries(read.filter((e) => e !== undefined)) + } let experiment = $derived(experiments.find((e) => e.id === experimentId)) @@ -120,6 +179,7 @@ runsLoadError = false try { experiments = await listSubjectExperiments() + await readCaseProgress() } catch (e) { runsLoadError = true sendUserToast(`Failed to load the runs: ${e}`, true) @@ -183,6 +243,8 @@ // Switching datasets leaves the previous request in flight; only the newest may write, or a // slow response for the dataset you just left replaces the one you are looking at. let loadGeneration = 0 + /** Which run the pane is opening; only the newest may clear `datasetLoading`. */ + let openGeneration = 0 async function loadDataset(path: string | undefined): Promise { const generation = ++loadGeneration @@ -345,6 +407,7 @@ await loadResults() } else { experiments = await listSubjectExperiments() + await readCaseProgress() } } finally { refreshing = false @@ -354,27 +417,46 @@ /** Opens a run, bringing its dataset with it and offering the run before it as the baseline. * Reading the cells is left to the effect on the selection, so every way in opens one alike. */ async function openRun(id: string) { - // Against the dataset that is loaded, not the one that is selected: skipping on the selection - // alone would leave a run open over a dataset whose cases and scorers were never read. const target = experiments.find((e) => e.id === id) - if (target && target.dataset !== dataset?.path) { - await useDataset(target.dataset) + // Only when the run itself changes: re-showing the one already open — arrowing back into it + // from the list — must keep whatever comparison the user picked. + if (id !== experimentId) { + const index = experiments.findIndex((e) => e.id === id) + // The run before it *of the same dataset*: the list spans datasets, and a run of another + // set of cases is not a baseline for this one. + baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id } - const index = experiments.findIndex((e) => e.id === id) - // The run before it *of the same dataset*: the list spans datasets, and a run of another set - // of cases is not a baseline for this one. - baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id experimentId = id - viewingRun = true selectedCaseId = undefined + // Opened first, read second: the dataset is a request, and waiting on it here is a click + // that does nothing at all until the network answers. The page carries the wait instead. + // + // Against what is *selected* as well as what is loaded. `selectedDataset` moves the moment a + // read starts, so a load still in flight for another dataset shows up here: without that + // test, opening a run of the dataset already committed would skip `useDataset` entirely and + // leave the in-flight one free to commit its cases under this run. + const needsDataset = + !!target && (target.dataset !== dataset?.path || target.dataset !== selectedDataset) + datasetLoading = needsDataset + viewingRun = true + if (needsDataset) { + // Numbered like `loadDataset`'s own read, and for the same reason: opening a second run + // while the first is still loading leaves two `finally`s racing, and the loser clearing + // the flag would uncover the table with neither dataset in hand. + const generation = ++openGeneration + try { + await useDataset(target!.dataset) + } finally { + if (generation === openGeneration) datasetLoading = false + } + } } async function runAll(runSubject: EvalSubject, path: string): Promise { if (!ws || !path) return false running = true - let id: string try { - id = await AiEvalsService.runExperiment({ + await AiEvalsService.runExperiment({ workspace: ws, requestBody: { dataset: path, subject: runSubject } }) @@ -386,9 +468,10 @@ // From here the run exists and is billing: what can still fail is reading it back, and // saying "failed to run" to that invites a second, duplicate run. try { + // Onto the list rather than into the run: a run that has just started has no answers and + // no scores, and the list already fills its row in as they land. Reading it is a click. if (path !== dataset?.path) await useDataset(path) await loadRuns() - await openRun(id) } catch (e) { sendUserToast( `The run started but could not be read back: ${e}. Reload the runs list to see it.`, @@ -407,8 +490,6 @@ /** The dataset is gone and every run of it with it: back to the list, on no dataset. */ async function datasetDeleted(path: string) { - // A run dialog waiting behind the drawer has nothing to come back to. - resumeRunDialog = false if (selectedDataset === path) { viewingRun = false selectedCaseId = undefined @@ -442,6 +523,13 @@ } let selectedRow = $derived(displayRows.find((row) => row.case_id === selectedCaseId)) + /** The case the side panel is showing. Held rather than read straight off the selection: the + * pane animates shut over a few hundred milliseconds, and the selection is gone on the first + * of them, which would empty the panel before it had finished closing. */ + let openRow = $state(undefined) + $effect(() => { + if (selectedRow) openRow = selectedRow + }) $effect(() => { if (!ws) return @@ -570,34 +658,54 @@
    + {#if loaded && loadError} +
    + Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
    + {:else} + + { + // Right opens the run under the highlight, falling back to whichever was open before; + // left is the way back, the same as the breadcrumb. + if (key === 'run') { + // Both branches go through `openRun`: it is what brings the run's own dataset back, + // and the fallback run may be of a dataset the list has since moved off. + const id = highlightedRunId ?? experimentId + if (id) openRun(id) + } else if (key === 'list') { + viewingRun = false + selectedCaseId = undefined + } + }} + pages={[ + { key: 'list', content: listPage }, + { key: 'run', content: runPage } + ]} + /> + {/if} +
    + +{#snippet listPage()} + +

    + Each run answers a dataset of cases with this agent and scores the answers, so runs can be + compared. +

    - {#if viewingRun} - - {/if}
    - {#if viewingRun && experiment?.run_job_id} - - Open the job - - - {/if} - {#if !viewingRun && loaded && datasets.length > 0} + {#if loaded && datasets.length > 0} -
    - {:else if !viewingRun || !loaded} - openRun(e.id)} - onEditDataset={async (path) => { - if (await useDataset(path)) datasetDrawer?.openDrawer('edit') - }} - onNew={() => (runDialogOpen = true)} - /> - {:else} - -
    + +
    ` — @@ -71,7 +73,7 @@ // query, and the user needs to configure it before pipeline scripts // can write to it. let datatables = resource( - () => $workspaceStore, + () => $operatingWorkspace, async (ws) => { if (!ws) return [] as string[] try { @@ -117,9 +119,9 @@ () => [input, refreshKey], async ([_input]) => { colDefsError = undefined - if (!_input || !$workspaceStore) return undefined + if (!_input || !$operatingWorkspace) return undefined try { - return await loadAllTablesMetaData($workspaceStore, _input) + return await loadAllTablesMetaData($operatingWorkspace, _input) } catch (e) { colDefsError = (e as Error)?.message || String(e) return undefined @@ -138,12 +140,12 @@ }) let dbTableOps = $derived( - input && tableColDefs && parsed.table && $workspaceStore + input && tableColDefs && parsed.table && $operatingWorkspace ? dbTableOpsWithPreviewScripts({ input, tableKey: parsed.table, colDefs: tableColDefs, - workspace: $workspaceStore + workspace: $operatingWorkspace }) : undefined ) diff --git a/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte new file mode 100644 index 0000000000..a1c769d9af --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte @@ -0,0 +1,50 @@ + + +{#if columns.length > 0} +
    +
    {analyzed ? 'columns' : 'columns declared'}
    +
    + {#each columns as col (col.name)} +
    + {col.name} + {#if col.type} + {col.type} + {/if} + {col.description} +
    + {/each} +
    + + {#if !analyzed} +
    + Declared metadata. Set `column_lineage: true` in the descriptor for the real column schema, + typed and in the order the model produces it. +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte index f21ca486af..d2d3c03b8a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte @@ -9,7 +9,6 @@ // ("This partition") or show the full table ("Whole table") via a toggle. import DBTable from '$lib/components/DBTable.svelte' import { resource } from 'runed' - import { workspaceStore } from '$lib/stores' import { loadAllTablesMetaData } from '$lib/components/apps/components/display/dbtable/metadata' import { dbTableOpsWithPreviewScripts } from '$lib/components/dbOps' import type { DbInput } from '$lib/components/dbTypes' @@ -18,6 +17,9 @@ import { twMerge } from 'tailwind-merge' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Full asset URI, e.g. `ducklake://main/orders_daily`. @@ -61,9 +63,9 @@ () => [input, refreshKey] as const, async ([_input]) => { colDefsError = undefined - if (!_input || !$workspaceStore) return undefined + if (!_input || !$operatingWorkspace) return undefined try { - return await loadAllTablesMetaData($workspaceStore, _input) + return await loadAllTablesMetaData($operatingWorkspace, _input) } catch (e) { colDefsError = (e as Error)?.message || String(e) return undefined @@ -89,12 +91,12 @@ }) let dbTableOps = $derived.by(() => { - if (!(input && tableColDefs && tableKey && $workspaceStore)) return undefined + if (!(input && tableColDefs && tableKey && $operatingWorkspace)) return undefined const ops = dbTableOpsWithPreviewScripts({ input, tableKey, colDefs: tableColDefs, - workspace: $workspaceStore, + workspace: $operatingWorkspace, whereClause }) // Read-only preview: drop the mutation handlers so DBTable hides its diff --git a/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte index 5afb51fe66..35eff3beae 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte @@ -7,7 +7,6 @@ // hand is shown above the grid so the affordance is self-documenting. import DBTable from '$lib/components/DBTable.svelte' import { resource } from 'runed' - import { workspaceStore } from '$lib/stores' import { fetchDucklakeColumnsAtVersion, dbTableOpsWithPreviewScripts @@ -17,6 +16,9 @@ import { AlertTriangle, Loader2, ClipboardCopy } from 'lucide-svelte' import { Button } from '$lib/components/common' import { twMerge } from 'tailwind-merge' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Full asset URI, e.g. `ducklake://main/orders_daily`. @@ -56,9 +58,10 @@ let columns = resource( () => [ducklake, tableKey, version] as const, async ([_ducklake, _tableKey, _version]) => { - if (!_ducklake || !_tableKey || _version == undefined || !$workspaceStore) return undefined + if (!_ducklake || !_tableKey || _version == undefined || !$operatingWorkspace) + return undefined const colDefs = await fetchDucklakeColumnsAtVersion({ - workspace: $workspaceStore, + workspace: $operatingWorkspace, ducklake: _ducklake, tableKey: _tableKey, version: _version @@ -73,13 +76,13 @@ let tableColDefs = $derived(ready ? columns.current!.colDefs : undefined) let dbTableOps = $derived.by(() => { - if (!(input && tableColDefs && tableKey && $workspaceStore && version != undefined)) + if (!(input && tableColDefs && tableKey && $operatingWorkspace && version != undefined)) return undefined const ops = dbTableOpsWithPreviewScripts({ input, tableKey, colDefs: tableColDefs, - workspace: $workspaceStore, + workspace: $operatingWorkspace, version }) // Historical reads are immutable: drop every mutation handler so DBTable diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 3439fa5ad5..b71a51ea0a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -17,7 +17,9 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode, DbtAssetProvenance } from './types' + PipelineMode, + DbtAssetProvenance + } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +78,9 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, + selectionColumnFailed = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -180,8 +185,16 @@ * the selected node's source on live-reload. */ localScriptsVersion?: unknown selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> - /** Transitive column-lineage trace for a selected ducklake asset (route page). */ + /** Transitive column-lineage trace for the selected asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** That trace still being fetched — a dbt relation's is a request of its + * own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** That trace cut at the part nearest the selection. */ + selectionColumnTruncated?: boolean + /** That trace could not be fetched. Distinguished from an empty one: a + * project without the analysis pass draws nothing either. */ + selectionColumnFailed?: boolean /** dbt provenance of the selected relation — carries its SQL. */ selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean @@ -514,6 +527,9 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionColumnLoading} + {selectionColumnTruncated} + {selectionColumnFailed} {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte index e3c6c04378..e341707250 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte @@ -28,6 +28,7 @@ @@ -22,6 +24,7 @@ on:click={() => (dispatch('close'), onClick?.())} on:pointerdown={(e) => e.stopPropagation()} {id} + {title} startIcon={{ icon: Icon ?? X }} iconOnly unifiedSize="sm" diff --git a/frontend/src/lib/components/common/EditableTextarea.svelte b/frontend/src/lib/components/common/EditableTextarea.svelte new file mode 100644 index 0000000000..26a6574027 --- /dev/null +++ b/frontend/src/lib/components/common/EditableTextarea.svelte @@ -0,0 +1,173 @@ + + + +{#if editing} + + +{:else} + + +{/if} diff --git a/frontend/src/lib/components/common/alert/Alert.svelte b/frontend/src/lib/components/common/alert/Alert.svelte index c3d86693be..e4cdd7bd7d 100644 --- a/frontend/src/lib/components/common/alert/Alert.svelte +++ b/frontend/src/lib/components/common/alert/Alert.svelte @@ -71,7 +71,9 @@ style={bgStyle} >
    -
    + +
    void href?: string icon?: any @@ -146,6 +147,7 @@ const items = typeof menuItems === 'function' ? menuItems() : menuItems return items.map((item) => ({ displayName: item.label, + description: item.description, action: item.onClick ? (e) => item.onClick?.(e) : undefined, icon: item.icon, disabled: item.disabled ?? false, diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index 56244632ef..524817ba34 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -23,6 +23,12 @@ /** Tailwind z-index class for the modal root. Override to stack this modal * above another modal that's already open (both default to `z-[9999]`). */ zIndexClass?: string + /** Render into `body` instead of where this component sits. Needed when an ancestor + * creates a stacking context the dialog has to escape — a drawer paints over the page + * whatever the dialog's z-index, and a `transform`, `filter` or `overflow` on the way + * up confines it. Off by default: it moves the dialog out of its DOM position, so opt + * in per call site rather than assuming every caller wants it. */ + alwaysPortal?: boolean children?: Snippet onConfirmed?: () => void | Promise onCanceled?: () => void @@ -40,6 +46,7 @@ id, trashbin = false, zIndexClass = 'z-[9999]', + alwaysPortal = false, children, onConfirmed, onCanceled @@ -141,7 +148,11 @@ - + {#if open}
    {#if showIcon} +
    {/if} -
    + + +

    {title}

    diff --git a/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte index 8d41fff902..bd830de1b7 100644 --- a/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte @@ -11,6 +11,15 @@ confirmCallback: () => void deployedBy?: string | undefined open?: boolean + /** Takes a drawer opening on the host editor's behalf, so the diff below is one the + * editor can take back down. Override anyway deploys that editor's content: without + * this the drawer would outlive it (a relocation remounts the editor) and deploy it + * at a path it has left. Omit where the host cannot be remounted under the drawer. */ + claimOpening?: () => number | undefined + /** The host could not read the deployed head, so this confirmation is caution rather + * than an observed newer version: nobody may have deployed over the user, and + * `deployedBy` is then whoever wrote the head, possibly themselves. */ + headUnknown?: boolean } let { @@ -19,13 +28,15 @@ diffDrawer = undefined, confirmCallback, deployedBy = undefined, - open = $bindable(false) + open = $bindable(false), + claimOpening = undefined, + headUnknown = false }: Props = $props() { open = false @@ -33,7 +44,11 @@ on:confirmed={() => confirmCallback()} >
    - A new version was deployed while you were editing this one. + + {headUnknown + ? 'This editor could not check whether a newer version is deployed, so it cannot tell whether this overwrites newer work.' + : 'A new version was deployed while you were editing this one.'} + {#if diffDrawer} diff --git a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte new file mode 100644 index 0000000000..5a51b4cfba --- /dev/null +++ b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte @@ -0,0 +1,326 @@ + + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
    + {#if draftTriggers.length > 0} +
    +
    + {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
    + +
    5 ? 'h-[300px]' : ''}> + +
    + + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
    +
    + Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
    + +
    5 ? 'h-[300px]' : ''}> + +
    + + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte b/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte index 26366325b5..20c29286b5 100644 --- a/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte +++ b/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte @@ -13,6 +13,10 @@ import type { UserDraftItemKind } from '$lib/gen' import DraftSyncConflictModal from './DraftSyncConflictModal.svelte' import OtherUsersDraftsModal, { type OtherDraftUser } from './OtherUsersDraftsModal.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { sendUserToast } from '$lib/toast' + import { base } from '$app/paths' + import { goto } from '$app/navigation' import StaleDraftModal from './StaleDraftModal.svelte' import ConfirmationModal from './ConfirmationModal.svelte' import { OtherUserDraftLoad } from '$lib/components/otherUserDraftLoad.svelte' @@ -38,15 +42,27 @@ draftSavedAt?: string | undefined /** ISO timestamp of the latest deploy at this path. */ deployedAt?: string | undefined - /** Precise staleness inputs (flows/apps): the deployed version the draft was - * forked from, and the current deployed head. When both are set they drive - * `isStale` and the dedup key instead of the timestamps — exact, and stable - * across autosaves (the timestamp drifts past `deployedAt` as you keep - * editing). Absent (pre-feature drafts, scripts) ⇒ timestamp fallback. */ - draftBaseVersion?: number | undefined - deployedHeadVersion?: number | undefined + /** Precise staleness inputs: the deployed version the draft forked from + * (`draft_base` on the get-by-path response) and the current deployed head, + * both as text whatever the kind. When both are set they drive `isStale` and + * the dedup key instead of the timestamps, which drift past `deployedAt` as + * you keep editing. Absent (a draft never forked from a deploy) ⇒ timestamp + * fallback. */ + draftBaseVersion?: string | undefined + deployedHeadVersion?: string | undefined + /** Who deployed the head, named in the stale prompt. */ + deployedBy?: string | undefined /** Discard the draft and reload deployed (same as "Reset to deployed"). */ onLoadLatestDeploy?: () => void | Promise + /** Opens the editor's Deployed↔Current diff from the stale prompt, so the + * choice between keeping and discarding is informed. Omit where the editor + * has no diff drawer; the action is then not rendered. */ + onViewDiff?: () => void | Promise + /** Runs before this editor follows its draft to the item's new path: the + * editor's own draft save, which materializes text the code editor still + * holds. Without it, keystrokes typed since the relocating save are lost + * to the navigation. */ + onBeforeRelocate?: () => void | Promise /** Defaults to true; set to false to suppress all modals. */ enabled?: boolean } @@ -65,7 +81,10 @@ deployedAt = undefined, draftBaseVersion = undefined, deployedHeadVersion = undefined, + deployedBy = undefined, onLoadLatestDeploy, + onViewDiff, + onBeforeRelocate, enabled = true }: Props = $props() @@ -74,9 +93,9 @@ let staleAlertKey = $state(undefined) let staleModalOpen = $state(false) - // Prefer the exact version comparison (flows/apps) over the timestamp: the - // draft's pinned fork base never drifts, whereas `draftSavedAt` advances past - // `deployedAt` once you keep editing a stale draft, hiding the staleness. + // Prefer the version comparison over the timestamp for every kind that supplies + // one: `draftSavedAt` advances past `deployedAt` as you keep editing, hiding the + // staleness outright. const useVersion = $derived(draftBaseVersion != null && deployedHeadVersion != null) const isStale = $derived( !!onLoadLatestDeploy && @@ -105,6 +124,48 @@ } }) }) + + const EDITOR_SEGMENT: Partial> = { + script: 'scripts/edit', + flow: 'flows/edit', + app: 'apps/edit', + raw_app: 'apps_raw/edit' + } + + // The item was moved while this editor was open: the draft row followed it + // and the save just landed there. Follow it too — the route reloads the item + // at its new path, and the stale prompt above then says what changed. Edits + // typed since that save are flushed first, so leaving this path drops none. + $effect(() => { + if (!enabled || !workspace || !path) return + const seg = EDITOR_SEGMENT[itemKind] + if (!seg) return + const query = { workspace, itemKind, path } + // The flush below saves again and can land here a second time, and a second move + // can land while it runs: the last destination reported is the one to follow. + let relocating = false + let destination: string | undefined = undefined + return UserDraftDbSyncer.onRelocated(query, async (newPath) => { + destination = newPath + if (relocating) return + relocating = true + await onBeforeRelocate?.() + await UserDraftDbSyncer.flush(query) + // `flush` resolves on a failed or rejected save as well, and leaving the + // route drops what it was carrying: stay, so the editor keeps the edits + // and its own failure indicator. + if ( + UserDraftDbSyncer.getState(query).failureMessage || + UserDraftDbSyncer.getConflict(query).conflict + ) { + relocating = false + return + } + const target = destination ?? newPath + sendUserToast(`This item was moved to ${target}. You are now editing it there.`) + await goto(`${base}/${seg}/${target}`) + }) + }) {#if enabled && workspace && path} @@ -130,9 +191,14 @@ {#if onLoadLatestDeploy} {/if} - import ConfirmationModal from './ConfirmationModal.svelte' - import { createEventDispatcher, untrack } from 'svelte' - import type { Trigger } from '$lib/components/triggers/utils' - import DataTable from '$lib/components/table/DataTable.svelte' - import { twMerge } from 'tailwind-merge' - import TriggerLabel from '$lib/components/triggers/TriggerLabel.svelte' - import { triggerIconMap } from '$lib/components/triggers/utils' - import { Star } from 'lucide-svelte' - import ToggleButtonGroup from '../toggleButton-v2/ToggleButtonGroup.svelte' - import ToggleButton from '../toggleButton-v2/ToggleButton.svelte' - import { userStore } from '$lib/stores' - import Badge from '../badge/Badge.svelte' - - interface Props { - open?: boolean - draftTriggers?: Trigger[] - isFlow?: boolean - } - - let { open = $bindable(false), draftTriggers = [], isFlow = false }: Props = $props() - - let selectedTriggers: Trigger[] = $state(untrack(() => draftTriggers)) - - const dispatch = createEventDispatcher<{ - canceled: void - confirmed: { selectedTriggers: Trigger[] } - }>() - - function toggleTrigger(trigger: Trigger, selected: 'discard' | 'deploy') { - if (selected === 'discard') { - if (trigger.isDraft) { - selectedTriggers = selectedTriggers.filter((t) => !t.isDraft || t.id !== trigger.id) - } else { - selectedTriggers = selectedTriggers.filter( - (t) => t.isDraft || t.type !== trigger.type || t.path !== trigger.path - ) - } - } else if (!isSelected(selectedTriggers, trigger)) { - selectedTriggers = [...selectedTriggers, trigger] - } - } - - function isSelected(triggers: Trigger[], trigger: Trigger): boolean { - if (trigger.isDraft) { - return triggers.some((t) => t.id === trigger.id) - } else { - return triggers.some((t) => t.path === trigger.path && t.type === trigger.type) - } - } - - function checkSavePermissions(trigger: Trigger) { - // Creating http trigger is forbidden for non-admin users - const adminOnly = - trigger.type === 'http' && - !$userStore?.is_admin && - !$userStore?.is_super_admin && - trigger.isDraft - - const invalidConfig = !trigger.draftConfig?.canSave - - return invalidConfig ? 'invalid-config' : adminOnly ? 'admin-only' : 'deploy' - } - - $effect(() => { - open && - (selectedTriggers = [...draftTriggers].filter((t) => checkSavePermissions(t) === 'deploy')) - }) - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
    -
    - {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
    - -
    5 ? 'h-[300px]' : ''}> - -
    - - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/common/confirmationModal/StaleDraftModal.svelte b/frontend/src/lib/components/common/confirmationModal/StaleDraftModal.svelte index cc027c6f12..9f4939eb2b 100644 --- a/frontend/src/lib/components/common/confirmationModal/StaleDraftModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/StaleDraftModal.svelte @@ -1,41 +1,66 @@ + +{#snippet body()} +
    + {#if icon} +
    {@render icon()}
    + {/if} +
    +
    + {@render title()} +
    + {#if subtitle} + + + {@render subtitle()} + + {/if} +
    +
    +{/snippet} + +{#if trailing} + +
    + {#if onClick} + + + {:else} + +
    {@render body()}
    + {/if} + {@render trailing()} +
    +{:else if !onClick} + + +
    + {@render body()} +
    +{:else} + +{/if} diff --git a/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts new file mode 100644 index 0000000000..c7325698e3 --- /dev/null +++ b/frontend/src/lib/components/common/listRow/listHighlight.svelte.ts @@ -0,0 +1,97 @@ +import { untrack } from 'svelte' + +/** + * The highlighted row of a searchable list: the one the arrow keys move and Enter + * activates, rendered by passing `highlighted` to `ListRow`. + * + * Pairs with a search field above the list — the arrows and Enter are answered while + * focus stays in it, so a query and a choice are one uninterrupted sequence. + */ +export function useListHighlight(opts: { + /** How many rows the list holds right now. */ + count: () => number + /** The DOM id of the row at this index — the same `id` given to its `ListRow`. */ + rowId: (index: number) => string + /** Where the highlight belongs when the list changes underneath it: the top hit while + * a search is on, and typically -1 (nothing lit) when it is not. */ + restingIndex: () => number + /** Open the row at this index. */ + onActivate: (index: number) => void + /** Ids of the elements whose Enter also activates the highlighted row — the search + * field. A focused row activates itself, so it is not one of these. */ + activateEnterFrom?: string[] +}) { + let index = $state(-1) + // Scrolling rows under a resting pointer makes the browser fire `mouseenter` on each + // one, which would drag the highlight back under the cursor as the arrow keys move it. + // Only a real pointer move hands the highlight back to the mouse. + let pointerOwns = $state(true) + + // Filtering reshuffles the rows under the highlight, so it goes back where the caller + // says it belongs rather than staying on a position that now means another row. + $effect(() => { + opts.count() + const resting = opts.restingIndex() + untrack(() => (index = resting)) + }) + + function land(next: number) { + const count = opts.count() + if (count === 0 || next < 0 || next >= count) return + pointerOwns = false + // Rows are tabbable, so focus can sit on one. Enter then activates whatever is + // focused, which has to stay the highlighted row — so any row counts, not just + // the lit one. Tab from the search field lands on the first row while the + // highlight rests on the best match, and testing only the lit row would leave + // focus behind and activate the wrong one. + const focusedId = document.activeElement?.id + const rowWasFocused = + !!focusedId && Array.from({ length: count }, (_, i) => opts.rowId(i)).includes(focusedId) + index = next + const row = document.getElementById(opts.rowId(index)) + row?.scrollIntoView({ block: 'nearest' }) + if (rowWasFocused) row?.focus() + } + + function move(delta: number) { + const count = opts.count() + if (count === 0) return + land(index < 0 ? (delta > 0 ? 0 : count - 1) : (index + delta + count) % count) + } + + return { + get index() { + return index + }, + /** Step the highlight, for a list whose own keys move it beyond Up and Down — + * a tree stepping into the children a folder just revealed. */ + move, + /** Put the highlight on a row named outright, rather than a step from wherever + * it is — the row a caller's own key landed on, or the one that has focus. A + * step cannot say this: from nothing lit it can only reach an end of the list. */ + moveTo: land, + /** Wire to each row's `onMouseEnter`. */ + hovered(i: number) { + if (pointerOwns) index = i + }, + /** Wire to the list container's `onpointermove`. */ + pointerMoved() { + pointerOwns = true + }, + /** Wire to the container that holds the search field and the rows, so the keys are + * answered whichever of the two has focus. */ + onKeydown(e: KeyboardEvent) { + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { + e.preventDefault() + move(e.key === 'ArrowDown' ? 1 : -1) + } else if ( + e.key === 'Enter' && + opts.activateEnterFrom?.includes((e.target as HTMLElement)?.id) && + index >= 0 + ) { + e.preventDefault() + opts.onActivate(index) + } + } + } +} diff --git a/frontend/src/lib/components/common/modal/Modal.svelte b/frontend/src/lib/components/common/modal/Modal.svelte index 9cfa11b84e..bcc4f33607 100644 --- a/frontend/src/lib/components/common/modal/Modal.svelte +++ b/frontend/src/lib/components/common/modal/Modal.svelte @@ -11,7 +11,11 @@ + + + + + + +
    + {#each pages as page, i (page.key)} + {@const built = visited.includes(page.key) || warmed} + {#if built || page.placeholder} + + +
    + {#if built} + {@render page.content()} + {:else} + {@render page.placeholder?.()} + {/if} +
    + {/if} + {/each} +
    + + diff --git a/frontend/src/lib/components/common/overlayHost.svelte.ts b/frontend/src/lib/components/common/overlayHost.svelte.ts index 0e76709cbf..e556b51e67 100644 --- a/frontend/src/lib/components/common/overlayHost.svelte.ts +++ b/frontend/src/lib/components/common/overlayHost.svelte.ts @@ -55,12 +55,13 @@ export function overlayStack(): OverlayStack { * Reads context, so call it during component initialisation; call the returned getter * where the target is used, to stay reactive as the host element mounts. */ -export function overlayPortalTarget(fallback: string): () => HTMLElement | string { +export function overlayPortalTarget(fallback: string | (() => string)): () => HTMLElement | string { const host = getOverlayHost() return () => { + const selector = typeof fallback === 'function' ? fallback() : fallback const el = host?.el() - if (!el) return fallback - return el.querySelector(fallback) ?? el + if (!el) return selector + return el.querySelector(selector) ?? el } } @@ -72,3 +73,23 @@ export function overlayHostActive(): () => boolean { const host = getOverlayHost() return () => host?.active() ?? true } + +const TOPMOST_SURFACE_KEY = 'topmostSurface' + +/** + * Declare whether the surface enclosing this subtree is the one on top. Set by whatever owns the + * stacking — a dialog, a drawer — so content inside it can tell a key meant for itself from one + * meant for something opened over it. + */ +export function setTopmostSurface(isTopmost: () => boolean) { + setContext(TOPMOST_SURFACE_KEY, isTopmost) +} + +/** + * Whether the enclosing surface is on top. True when nothing declared otherwise, so content that + * is not inside such a surface is not silently made deaf. + */ +export function topmostSurface(): () => boolean { + const isTopmost = getContext<(() => boolean) | undefined>(TOPMOST_SURFACE_KEY) + return () => isTopmost?.() ?? true +} diff --git a/frontend/src/lib/components/common/popup/PopupV2.svelte b/frontend/src/lib/components/common/popup/PopupV2.svelte index 42257ff53b..fd0987390b 100644 --- a/frontend/src/lib/components/common/popup/PopupV2.svelte +++ b/frontend/src/lib/components/common/popup/PopupV2.svelte @@ -52,11 +52,6 @@ use:clickOutside={{ eventToListenName: 'pointerdown', stopPropagation: true, - exclude: async () => { - const tutorial = document.querySelector('#driver-popover-content') as HTMLElement | null - if (tutorial) return [tutorial] - return [] - }, onClickOutside: () => (open = false) }} > diff --git a/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte b/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte index 260e368e2e..d18df4c655 100644 --- a/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte +++ b/frontend/src/lib/components/common/sidebar/SidebarNavigation.svelte @@ -2,6 +2,7 @@ import type { ComponentType } from 'svelte' import { twMerge } from 'tailwind-merge' import Button from '$lib/components/common/button/Button.svelte' + import { arrowTabNav } from '$lib/attachments/arrowTabNav' import EEOnly from '$lib/components/EEOnly.svelte' import { enterpriseLicense } from '$lib/stores' @@ -32,7 +33,10 @@ let { groups, selectedId, onNavigate, class: className = '' }: Props = $props() -
    + +
    {#each groups as group (group.title)}
    {#if group.title} diff --git a/frontend/src/lib/components/common/table/AppRow.svelte b/frontend/src/lib/components/common/table/AppRow.svelte index 636bd2c9d6..bd881a31b7 100644 --- a/frontend/src/lib/components/common/table/AppRow.svelte +++ b/frontend/src/lib/components/common/table/AppRow.svelte @@ -11,7 +11,7 @@ import { createEventDispatcher } from 'svelte' import Button from '../button/Button.svelte' import Row from './Row.svelte' - import type { RowSelection } from './rowSelection' + import { selectMenuItems, type RowSelection } from './rowSelection' import InheritedLabels from '$lib/components/InheritedLabels.svelte' import Badge from '../badge/Badge.svelte' import { @@ -109,13 +109,13 @@ ? `${base}/apps${app.raw_app ? '_raw' : ''}/edit/${app.path}` : `${base}/apps${app.raw_app ? '_raw' : ''}/get/${app.path}`} kind="app" + {keyboardSelected} {marked} path={(app as any).draft_path ?? app.path} summary={app.is_draft ? `${app.summary || (app as any).draft_path || app.path}*` : app.summary} workspaceId={app.workspace_id ?? $workspaceStore ?? ''} canFavorite={!app.draft_only} {depth} - {keyboardSelected} {rowSelection} > {#snippet badges()} @@ -188,6 +188,21 @@ const canEdit = canWrite && showEditButton if (draft_only) { return [ + ...selectMenuItems(rowSelection), + { + displayName: 'Move/Rename', + icon: FolderOpen, + action: () => { + // Addressed by the generated path its draft row sits at, but + // named by the path typed in the editor. + moveDrawer.openDrawer((app as any).draft_path ?? path, summary, 'app', { + storagePath: path, + rawApp: !!app.raw_app + }) + }, + disabled: !showEditButton, + hide: $userStore?.operator + }, { displayName: 'Delete', icon: Trash, @@ -221,6 +236,7 @@ ] } return [ + ...selectMenuItems(rowSelection), { displayName: 'Duplicate/Fork', icon: GitFork, diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index a155ca3a97..f9af803694 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -14,7 +14,7 @@ import Badge from '../badge/Badge.svelte' import Button from '../button/Button.svelte' import Row from './Row.svelte' - import type { RowSelection } from './rowSelection' + import { selectMenuItems, type RowSelection } from './rowSelection' import { sendUserToast } from '$lib/toast' import { copyToClipboard, isOwner } from '$lib/utils' import { isDeployable } from '$lib/utils_deployable' @@ -35,6 +35,7 @@ HistoryIcon } from 'lucide-svelte' import FlowHistory from '$lib/components/flows/FlowHistory.svelte' + import ChatFlowBadge from '$lib/components/flows/ChatFlowBadge.svelte' import InheritedLabels from '$lib/components/InheritedLabels.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' import { editInForkAllowed, editInForkLabel, onEditInForkClick } from '$lib/utils/editInFork' @@ -48,6 +49,9 @@ draft_path?: string draft_users?: { username?: string | null }[] canWrite: boolean + /** Projected from the flow value by the listing; a chat-input flow opens + * as a conversation and is badged as such. */ + chat_input_enabled?: boolean } marked: string | undefined shareModal: ShareModal @@ -124,6 +128,10 @@ {/if} +{#snippet chatBadge()} + +{/snippet} + {#snippet badges()} {#if flow.archived} @@ -210,6 +219,20 @@ const canEdit = flow.canWrite && showEditButton if (draft_only) { return [ + ...selectMenuItems(rowSelection), + { + displayName: 'Move/Rename', + icon: FolderOpen, + action: () => { + // Addressed by the generated path its draft row sits at, but + // named by the path typed in the editor. + moveDrawer.openDrawer((flow as any).draft_path ?? path, flow.summary, 'flow', { + storagePath: path + }) + }, + disabled: !showEditButton, + hide: $userStore?.operator + }, { displayName: 'Delete', icon: Trash, @@ -233,6 +256,7 @@ ] } return [ + ...selectMenuItems(rowSelection), { displayName: 'View runs', icon: List, diff --git a/frontend/src/lib/components/common/table/RawAppRow.svelte b/frontend/src/lib/components/common/table/RawAppRow.svelte index 3f74356506..ff31eb1aea 100644 --- a/frontend/src/lib/components/common/table/RawAppRow.svelte +++ b/frontend/src/lib/components/common/table/RawAppRow.svelte @@ -35,13 +35,13 @@ {#snippet badges()} diff --git a/frontend/src/lib/components/common/table/Row.svelte b/frontend/src/lib/components/common/table/Row.svelte index 849d77e2d9..4fb90151c1 100644 --- a/frontend/src/lib/components/common/table/Row.svelte +++ b/frontend/src/lib/components/common/table/Row.svelte @@ -13,6 +13,8 @@ interface Props { marked: string | undefined selected?: boolean + /** Highlighted by the list's keyboard arrow-navigation (distinct from `selected`, + * which is the checkbox multi-select state). Scrolls itself into view. */ keyboardSelected?: boolean disabled?: boolean canFavorite?: boolean @@ -65,6 +67,9 @@ badges?: import('svelte').Snippet actions?: import('svelte').Snippet customSummary?: import('svelte').Snippet + /** Rendered inline right after the title, unlike `badges`, which sit in + * their own column and are hidden below `lg`. */ + titleBadge?: import('svelte').Snippet /** Overrides the secondary path line (e.g. to strike a renamed path). * Falls back to the plain `path` string when not provided. */ pathDisplay?: import('svelte').Snippet @@ -99,6 +104,7 @@ badges, actions, customSummary, + titleBadge, pathDisplay, onSelect = () => {} }: Props = $props() @@ -273,7 +279,12 @@
    {/if}
    -
    +
    {#if customSummary} {@render customSummary?.()} {:else if marked} @@ -281,6 +292,7 @@ {:else} {!summary || summary.length == 0 ? displayPath : summary} {/if} + {@render titleBadge?.()}
    {#if pathDisplay} diff --git a/frontend/src/lib/components/common/table/ScriptRow.svelte b/frontend/src/lib/components/common/table/ScriptRow.svelte index 25b557690e..938086a0e8 100644 --- a/frontend/src/lib/components/common/table/ScriptRow.svelte +++ b/frontend/src/lib/components/common/table/ScriptRow.svelte @@ -9,14 +9,21 @@ import type ShareModal from '$lib/components/ShareModal.svelte' import { ScriptService, type Script } from '$lib/gen' - import { userStore, userWorkspaces, workspaceStore } from '$lib/stores' + import { + disableHubStore, + hubBaseUrlStore, + userStore, + userWorkspaces, + workspaceStore + } from '$lib/stores' + import { scriptToHubUrl } from '$lib/hub' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { createEventDispatcher } from 'svelte' import Badge from '../badge/Badge.svelte' import Button from '../button/Button.svelte' import Row from './Row.svelte' - import type { RowSelection } from './rowSelection' + import { selectMenuItems, type RowSelection } from './rowSelection' import { sendUserToast } from '$lib/toast' import { capitalize, copyToClipboard, isOwner } from '$lib/utils' import { isDeployable } from '$lib/utils_deployable' @@ -32,6 +39,7 @@ FolderOpen, ChevronUpSquare, GitFork, + Globe2, List, Pen, Shield, @@ -47,7 +55,12 @@ import Popover from '$lib/components/Popover.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' - import { editInForkAllowed, editInForkLabel, onEditInForkClick } from '$lib/utils/editInFork' + import { + claimTab, + editInForkAllowed, + editInForkLabel, + onEditInForkClick + } from '$lib/utils/editInFork' import EditInForkButton from './EditInForkButton.svelte' import { isCloudHosted } from '$lib/cloud' @@ -147,6 +160,7 @@ ? `${base}/scripts/edit/${script.path}` : `${base}/scripts/get/${script.hash}?workspace=${$workspaceStore}`} kind="script" + {keyboardSelected} {marked} path={script.draft_path ?? script.path} summary={script.is_draft @@ -156,7 +170,6 @@ workspaceId={$workspaceStore ?? ''} canFavorite={!script.draft_only} {depth} - {keyboardSelected} {rowSelection} > {#snippet badges()} @@ -187,7 +200,9 @@ CI test {/if} - {#if script.kind !== 'script'} + + {#if script.kind && script.kind !== 'script'} {script.kind === 'failure' ? 'Error handler' : capitalize(script.kind)} @@ -266,6 +281,7 @@ const canEdit = script.canWrite && showEditButton if (script.draft_only) { return [ + ...selectMenuItems(rowSelection), { displayName: 'View code', icon: Code, @@ -273,6 +289,19 @@ showCode(script.path, script.summary) } }, + { + displayName: 'Move/Rename', + icon: FolderOpen, + action: () => { + // Addressed by the generated path its draft row sits at, but + // named by the path typed in the editor. + moveDrawer.openDrawer(script.draft_path ?? script.path, script.summary, 'script', { + storagePath: script.path + }) + }, + disabled: !showEditButton, + hide: $userStore?.operator + }, { displayName: 'Delete', icon: Trash, @@ -296,6 +325,7 @@ ] } return [ + ...selectMenuItems(rowSelection), { displayName: 'View code', icon: Code, @@ -401,6 +431,32 @@ copyToClipboard(script.path) } }, + { + displayName: 'Publish to Hub', + icon: Globe2, + action: async () => { + // The row only carries metadata, so the code has to be fetched first; the tab is + // claimed before that, since Safari won't open one after an await. + const tab = claimTab() + try { + const fullScript = await ScriptService.getScriptByPath({ + workspace: $workspaceStore!, + path: script.path + }) + const url = scriptToHubUrl(fullScript, $hubBaseUrlStore).toString() + if (tab) { + tab.show(url) + } else if (!window.open(url)) { + sendUserToast('Allow popups to publish this script to the Hub', true) + } + } catch (e: any) { + tab?.discard() + sendUserToast(`Could not load ${script.path}: ${e?.body ?? e?.message ?? e}`, true) + } + }, + // Operators can't write scripts, so they have nothing to publish. + hide: $disableHubStore || $userStore?.operator + }, { displayName: script.archived ? 'Unarchive' : 'Archive', icon: Archive, diff --git a/frontend/src/lib/components/common/table/rowSelection.ts b/frontend/src/lib/components/common/table/rowSelection.ts index 081f29addc..a78fe0aa1d 100644 --- a/frontend/src/lib/components/common/table/rowSelection.ts +++ b/frontend/src/lib/components/common/table/rowSelection.ts @@ -1,3 +1,6 @@ +import { SquareCheckBig } from 'lucide-svelte' +import type { Item } from '$lib/utils' + /** * Wiring for a row whose kind icon doubles as a selection control: the icon * swaps to a checkbox on hover, and stays one while a selection is active. @@ -14,3 +17,19 @@ export type RowSelection = { active: boolean onToggle: (e: MouseEvent | KeyboardEvent) => void } + +/** + * The row menu's way into a selection, for the rows that offer one. The icon + * checkbox is the fast path but only appears on hover; this is the one a user + * can find by looking. + */ +export function selectMenuItems(rowSelection: RowSelection | undefined): Item[] { + if (!rowSelection) return [] + return [ + { + displayName: rowSelection.selected ? 'Deselect' : 'Select', + icon: SquareCheckBig, + action: (e) => rowSelection.onToggle(e) + } + ] +} diff --git a/frontend/src/lib/components/common/tabs/Tab.svelte b/frontend/src/lib/components/common/tabs/Tab.svelte index a652525b45..edb1135bbc 100644 --- a/frontend/src/lib/components/common/tabs/Tab.svelte +++ b/frontend/src/lib/components/common/tabs/Tab.svelte @@ -99,6 +99,7 @@ }} {disabled} {id} + data-tab-selected={isSelected ? 'true' : undefined} >
    ('[data-tab-selected="true"]') + if (el) lastX = el.offsetLeft + bar = { x: lastX, w: el ? el.offsetWidth : 0 } + } + + // Placing the bar for the first paint. Every later move comes from the observers below: a + // Tab marks itself selected in its own update, which has not run when an effect here does, + // so measuring from this side alone lands the bar on the tab that was selected before. + $effect(() => { + if (!slidingIndicator) return + void row + measureBar() + }) + + $effect(() => { + if (!slidingIndicator || !row) return + const ro = new ResizeObserver(measureBar) + ro.observe(row) + // The mark moving is the selection changing, and a tab added or removed changes what the + // bar has to sit on. Text counts too: a label rewritten in place — a count arriving, + // Result becoming Error — resizes the tab under the bar without touching the tree, and + // Svelte writes it straight to the node, so childList never sees it. + const mo = new MutationObserver(measureBar) + mo.observe(row, { + childList: true, + subtree: true, + characterData: true, + attributes: true, + attributeFilter: ['data-tab-selected'] + }) + return () => { + ro.disconnect() + mo.disconnect() + } + }) + let hashValues = $derived(values ? values.map((x) => '#' + x) : undefined) function hashChange() { @@ -77,10 +135,26 @@ {#if !hideTabs} - -
    +
    {@render children?.({ selected })} + {#if slidingIndicator} + + {/if}
    {/if} diff --git a/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte b/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte index 621e8383d0..2773503df9 100644 --- a/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte +++ b/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte @@ -7,6 +7,9 @@ interface Props { label?: string | undefined + /** Shown instead of `label` below the `xl` breakpoint, for groups that must keep + * their width inside a narrow table cell. The full label stays the accessible name. */ + shortLabel?: string | undefined iconOnly?: boolean tooltip?: string | undefined icon?: any | undefined @@ -30,6 +33,7 @@ let { label = undefined, + shortLabel = undefined, iconOnly = false, tooltip = undefined, icon = undefined, @@ -68,6 +72,7 @@ - {/snippet} - +

    AI can help with these inputs

    + + {#snippet fallback()} + + {/snippet} + +
    +
    +

    + {instructions + ? 'Instructions: ' + instructions + : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} +

    +
    -
    -

    - {instructions - ? 'Instructions: ' + instructions - : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} -

    -
    -
    +{/if} diff --git a/frontend/src/lib/components/copilot/AIFormSettings.svelte b/frontend/src/lib/components/copilot/AIFormSettings.svelte index 44151a01b3..ef627ace4b 100644 --- a/frontend/src/lib/components/copilot/AIFormSettings.svelte +++ b/frontend/src/lib/components/copilot/AIFormSettings.svelte @@ -3,6 +3,7 @@ import Label from '../Label.svelte' import Toggle from '../Toggle.svelte' import Tooltip from '../Tooltip.svelte' + import { copilotInfo } from '$lib/aiStore' interface Props { prompt?: string | undefined @@ -12,35 +13,37 @@ let { prompt = $bindable(undefined), type = 'script' }: Props = $props() -
    - { - if (prompt !== undefined) { - prompt = undefined - } else { - prompt = '' - } - }} - options={{ right: `Enable filling ${type} inputs with AI` }} - /> - {#if prompt !== undefined} -
    - -
    - {/if} -
    +{#if !$copilotInfo.workspaceDisabled} +
    + { + if (prompt !== undefined) { + prompt = undefined + } else { + prompt = '' + } + }} + options={{ right: `Enable filling ${type} inputs with AI` }} + /> + {#if prompt !== undefined} +
    + +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/ChatModelSettings.svelte b/frontend/src/lib/components/copilot/ChatModelSettings.svelte new file mode 100644 index 0000000000..a1e677d46b --- /dev/null +++ b/frontend/src/lib/components/copilot/ChatModelSettings.svelte @@ -0,0 +1,314 @@ + + +{#snippet trigger()} +
    + +
    +{/snippet} + +{#snippet typedField( + value: string, + placeholder: string, + onCommit: (value: string) => void, + close: () => void +)} + {#key value} + onCommit(e.currentTarget.value.trim()), + // Capture, not bubble: Svelte delegates `keydown` to the root, which sits above the + // menu — so a bubble handler here would run only after melt's own listener had read + // the key as typeahead and moved focus. A capture key is not delegatable, so this + // becomes a real listener on the input and sees the event first. + onkeydowncapture: (e) => { + // Escape cancels: let it reach the menu with the value untouched. + if (e.key === 'Escape') return + // Tab closes the menu, unmounting this field before focus moves, so no change + // event would ever fire. Commit on the way past. + if (e.key === 'Tab') { + onCommit(e.currentTarget.value.trim()) + return + } + // Enter means done: commit and close, rather than leaving the menu open around a + // field the commit is about to rebuild. + if (e.key === 'Enter') { + e.preventDefault() + onCommit(e.currentTarget.value.trim()) + close() + return + } + // Everything else is typing; the menu reads loose keys as typeahead. + e.stopPropagation() + } + }} + /> + {/key} +{/snippet} + +{#snippet section(sec: ChoiceSection, item: MeltItem, close: () => void)} +
    {sec.label}
    + {#if sec.loading} +
    + Loading... +
    + {:else if sec.options.length === 0} +
    {sec.emptyMessage ?? 'Nothing to choose from'}
    + {:else} +
    + {#each sec.options as option (option.key)} + option.onSelect()}> + {option.label} + {#if option.hint} + {option.hint} + {/if} + {#if option.selected} + + {/if} + + {/each} +
    + {/if} + {#if sec.custom && !sec.loading} + {@const custom = sec.custom} +
    + {@render typedField( + '', + custom.placeholder, + (value) => { + if (value) custom.onCommit(value) + }, + close + )} +
    + {/if} +{/snippet} + +{#snippet rows(items: Item[], item: MeltItem, builders: MeltBuilders)} + {#each items.filter((row) => !row.hide) as row (row.displayName)} + {#if row.separatorTop} +
    + {/if} + {#if row.submenuItems} + + + {:else} + row.action?.(e)}> + {#if row.icon} + + {/if} + {row.displayName} + {#if row.selected} + + {/if} + + {/if} + {/each} +{/snippet} + +{#if config.readOnly} + {@render trigger()} +{:else} + + {#snippet buttonReplacement()} + {@render trigger()} + {/snippet} + {#snippet menu({ item, builders, close })} +
    + {#if config.topItems} +
    + {@render rows(config.topItems(close), item, builders)} +
    + {/if} + {#each config.sections ?? [] as sec (sec.label)} +
    + {@render section(sec, item, close)} +
    + {/each} + {#if reasoning} +
    + {#if controlState === 'fixed'} + {}} + unsupportedReason={fixedReason} + /> + {:else if controlState === 'awaiting-model'} + {}} + unsupportedReason="Pick a model first" + /> + {:else if controlState === 'unknown'} + +
    +
    Thinking
    + {@render typedField(reasoning.value ?? '', 'none', reasoning.onSelect, close)} +
    + Windmill has no thinking levels for this provider — type what it accepts. +
    +
    + {:else if controlState === 'ladder'} + + effortSlider?.adjust(e)} + class="block group" + > + (stop === reasoning?.offToken ? 'off' : stop)} + overrideLabel={stops.includes(currentStop) ? undefined : effortLabel} + /> + + {:else} + + {}} + unsupportedReason="Not supported by this model" + /> + {/if} +
    + {/if} + {#if config.bottomItems} +
    + {@render rows(config.bottomItems(close), item, builders)} +
    + {/if} +
    + {/snippet} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index 6a8dc6892e..94715a0f18 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -79,66 +79,68 @@ }) - - {#snippet trigger()} -
    - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} -
    - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + /> +
    + {:else} +
    +

    Enable Windmill AI in the workspace settings

    +
    + {/if} +
    + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte b/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte new file mode 100644 index 0000000000..24ea4cf8aa --- /dev/null +++ b/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte @@ -0,0 +1,172 @@ + + +{#if unsupportedReason} + +
    +
    Thinking
    +
    {unsupportedReason}
    +
    +{:else} +
    + Thinking + {overrideLabel ?? format(current)} +
    + {#if stops.length > 1} + +
    + onSelect(stops[+e.currentTarget.value])} + onclick={(e) => { + // `click`, not `pointerup`: it is the event that means pressed and released on + // the track, so a press that began on the row above cannot commit an effort + // nobody chose. Only the click that moved nothing — any other stop has already + // committed through `oninput`, and doing it again would write it twice. + if (!hasPosition && +e.currentTarget.value === stopIndex) { + onSelect(stops[stopIndex]) + } + }} + use:isolatePointer + class="lean-range no-default-style w-full" + aria-label="Reasoning effort" + /> +
    + {/if} +{/if} + + diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index e61ab5cbf2..f399726988 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,5 +1,5 @@ - - {#snippet trigger()} - +{#if !$copilotInfo.workspaceDisabled} + + {#snippet trigger()} - - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} - - {/snippet} -
    + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + > + Generate + + + {:else} +
    +

    Enable Windmill AI in the workspace settings

    +
    + {/if} + + {/snippet} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/ScriptFix.svelte b/frontend/src/lib/components/copilot/ScriptFix.svelte index bbd9341700..78c4804cc8 100644 --- a/frontend/src/lib/components/copilot/ScriptFix.svelte +++ b/frontend/src/lib/components/copilot/ScriptFix.svelte @@ -13,6 +13,7 @@ import { getOpenInSessionHandoff } from '$lib/components/sessions/openInSessionContext' import { AIBtnClasses } from './chat/AIButtonStyle' import { getContext } from 'svelte' + import { logFeatureUsage } from '$lib/utils/featureUsage' let { lang, @@ -24,7 +25,7 @@ /** The failing run's error, used when there is no job to point at. */ error?: string /** The failing run's job id. Preferred over `error`: the chat reads the - * run itself with `get_job_logs`, which gives it the logs rather than + * run itself with `get_run`, which gives it the logs rather than * just the thrown value, and keeps the composer readable. */ jobId?: string /** Set when this sits in a flow step's preview, so the session opens on @@ -45,9 +46,28 @@ ? `Fix this error in ${what}:\n\n\`\`\`\n${error}\n\`\`\`` : `Fix the error from the last run of ${what}.` }) + // Anonymous counter for a failing run being handed to AI, keyed by where the run was. All + // three branches below report the same action: which one is on screen follows the session + // gate and whether a chat is already beside this panel, not a choice made here. + function logAiFix() { + logFeatureUsage('ai_fix', 'requested', { key: moduleId ? 'flow_step' : 'script' }) + } + const sessionSource = $derived.by(() => { const source = handoff?.source({ moduleId }) - return source ? { ...source, seedPrompt, autoSend: true } : undefined + if (!source) return undefined + // The counter wraps the editor's own hook rather than replacing it: that hook persists + // the draft the session opens on, so dropping it would fix an older copy of the code. + const editorBeforeOpen = source.beforeOpen + return { + ...source, + seedPrompt, + autoSend: true, + beforeOpen: async () => { + logAiFix() + await editorBeforeOpen?.() + } + } }) // Inside a session pane the chat is already beside this panel, so there is @@ -57,7 +77,7 @@ const sessionScopedManager = getContext('aiChatManager') -{#if SUPPORTED_LANGUAGES.has(lang)} +{#if SUPPORTED_LANGUAGES.has(lang) && !$copilotInfo.workspaceDisabled} {#if sessionScopedManager} - {:else} + {:else if !$copilotInfo.workspaceDisabled} togglePanel() })} -{:else} +{:else if !$copilotInfo.workspaceDisabled} {#snippet trigger()} {@render button({ onPress: () => togglePanel() })} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index 2a51b1b2e5..fd95613f2e 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -44,27 +44,43 @@ const isAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) const hasCopilot = $derived($copilotInfo.enabled) + // Another tab is running a turn on this session: transcript stays readable, + // composer locks, and the chat re-reads the shared record when the turn ends. + const runHeldElsewhere = $derived(aiChatManager.runHeldElsewhere) const disabled = $derived( forceDisabled || + runHeldElsewhere || !hasCopilot || (aiChatManager.mode === AIMode.SCRIPT && aiChatManager.scriptEditorOptions?.lang && !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang)) ) + // A spent free grant is not an unconfigured workspace: AIChatDisplay already shows an + // in-thread banner naming the real cause and linking to the key settings, so the generic + // "enable Windmill AI" line would both duplicate it and misstate why the chat is off. + const freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) const disabledMessage = $derived( forceDisabled ? forceDisabledMessage - : !hasCopilot - ? $aiUserDisabled - ? 'Windmill AI is disabled in your account settings' - : isAdmin - ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` - : 'Ask an admin to enable Windmill AI in this workspace to use this chat' - : aiChatManager.mode === AIMode.SCRIPT && - aiChatManager.scriptEditorOptions?.lang && - !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) - ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` - : '' + : runHeldElsewhere + ? // The typing indicator and the composer placeholder already carry + // this state; a footer note would say it a third time. + '' + : freeTierExhausted + ? '' + : !hasCopilot + ? $copilotInfo.workspaceDisabled + ? 'Windmill AI is hidden in this workspace' + : $aiUserDisabled + ? 'Windmill AI is disabled in your account settings' + : isAdmin + ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` + : 'Ask an admin to enable Windmill AI in this workspace to use this chat' + : aiChatManager.mode === AIMode.SCRIPT && + aiChatManager.scriptEditorOptions?.lang && + !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) + ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` + : '' ) const suggestions = [ diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c68945695b..cf27a45d89 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -1,4 +1,5 @@ +{#snippet freeTierExhaustedBanner()} +
    + +
    + + You have used all of your free Windmill AI tokens. Add your own API key to keep using AI. + + +
    +
    +
    +{/snippet} + +{#snippet freeTierUsageBanner()} +
    + + {freeTierUsedPct}% of your free Windmill AI used + + +
    +{/snippet} +
    {#each pastChats as chat (chat.id)} diff --git a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte index d38caf8f0a..c9c786ccbc 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte @@ -1,16 +1,16 @@ {#snippet externalLinkIcon()} {/snippet} - - {#snippet buttonReplacement()} -
    - -
    - {/snippet} - {#snippet menu({ item, builders, close })} -
    - - + -
    -
    Model
    -
    - {#each models as m (m.provider + m.model)} - selectModel(m)} - > - {m.model} - {#if m.model === providerModel.model && m.provider === providerModel.provider} - - {/if} - - {/each} -
    - -
    - {#if capability.supported} - - -
    - Thinking - {currentStop} -
    - {#if stops.length > 1} - -
    - selectReasoning(stops[+e.currentTarget.value])} - use:isolatePointer - class="lean-range no-default-style w-full" - aria-label="Reasoning effort" - /> -
    - {/if} -
    - {:else} - -
    -
    Thinking
    -
    Not supported by this model
    -
    - {/if} - - - (thinkingPreferences.expandByDefault = !thinkingPreferences.expandByDefault)} - > - Always expand thinking - {#if thinkingPreferences.expandByDefault} - - {/if} - -
    - {/snippet} -
    - - - - +{#if promptSettings} + +{/if} diff --git a/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte b/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte index 1a9df4816c..4f3ef368ad 100644 --- a/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte +++ b/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte @@ -7,6 +7,7 @@ import { userStore, workspaceStore } from '$lib/stores' import { chatState } from './sharedChatState.svelte' import { loadCopilot } from '$lib/components/copilot/loadCopilot' + import { copilotInfo } from '$lib/aiStore' import { aiChatManager } from './AIChatManager.svelte' import { onDestroy } from 'svelte' import Button from '$lib/components/common/button/Button.svelte' @@ -66,6 +67,14 @@ } }) + // The pane restores its last open state from localStorage before the config can say + // the workspace hid the assistant; close it as soon as that is known. + $effect(() => { + if ($copilotInfo.workspaceDisabled && chatState.size > 0) { + aiChatManager.closeChat() + } + }) + const historyManager = aiChatManager.historyManager historyManager.init() diff --git a/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte b/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte index d17d77af29..17cc2bb4c1 100644 --- a/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte @@ -4,7 +4,7 @@ import { CircleHelp, ArrowUp, Plus, Square, SquareCheck } from 'lucide-svelte' import Button from '$lib/components/common/button/Button.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' import type { UserQuestionDisplay } from './shared' // Sessions inject a per-pane `AIChatManager` via context; outside of @@ -12,7 +12,7 @@ // this, answers clicked inside a session would dispatch to the singleton's // pending callbacks map (which doesn't have the session manager's question // callback), and the AI loop would stall. - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() interface Props { toolCallId: string @@ -93,14 +93,14 @@ } return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [choice]) + chatHost.handleUserQuestionAnswer(toolCallId, [choice]) } function submitPicked() { if (!multiSelect || picked.size === 0) { return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [...picked]) + chatHost.handleUserQuestionAnswer(toolCallId, [...picked]) } function submitCustomAnswer() { @@ -119,7 +119,7 @@ return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [answer]) + chatHost.handleUserQuestionAnswer(toolCallId, [answer]) } function handleChoiceKeydown(event: KeyboardEvent, choice: string, index: number) { @@ -245,7 +245,7 @@ viewport bottom) so the mount scrollIntoView leaves the Submit button uncovered. -->
    diff --git a/frontend/src/lib/components/copilot/chat/AssistantFilesSection.svelte b/frontend/src/lib/components/copilot/chat/AssistantFilesSection.svelte new file mode 100644 index 0000000000..bae2f7bf32 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/AssistantFilesSection.svelte @@ -0,0 +1,158 @@ + + + +
    + {#snippet action()} + {#if total > 0} + + + {ready === total ? `${total} usable` : `${ready} of ${total} usable`} + + {/if} + {/snippet} + + {#if total === 0} + + {:else} +
    + {#each folders as folder (folder.name)} + {#snippet icon()} + + {/snippet} + {#snippet title()} + {folder.name} + {@const label = attachmentStatusLabel(folder.status)} + {#if label} + {label} + {/if} + {/snippet} + {#snippet subtitle()} + {folder.files.length} + {folder.files.length === 1 ? 'file' : 'files'} + {/snippet} + {#snippet trailing()} +
    + {/if} +
    + + pendingDetach && detach(pendingDetach)} + onCanceled={() => (pendingDetach = undefined)} +> + + The assistant loses access to {pendingDetach?.name}. Nothing + is deleted from your disk, and you can attach it again from the composer. + + diff --git a/frontend/src/lib/components/copilot/chat/AssistantInstructionsSection.svelte b/frontend/src/lib/components/copilot/chat/AssistantInstructionsSection.svelte new file mode 100644 index 0000000000..5d825bd5c8 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/AssistantInstructionsSection.svelte @@ -0,0 +1,358 @@ + + + + + +{#snippet field(p: { value: string; readOnly: boolean; onInput: (v: string) => void })} +
    + p.onInput(e.currentTarget.value) + }} + /> + {#if !p.readOnly} + + {p.value.length}/{MAX_PROMPT_LENGTH} characters + + {/if} +
    +{/snippet} + +
    + {#snippet action()} +
    + {#if isAdmin} + + {/if} + +
    + {/snippet} + + + + + {#snippet content()} + +
    + {note} + {#if workspaceReadOnly && tab === 'workspace'} + + {readOnlyReason} + + {/if} + + + {@render field({ + value: workspaceDraft, + readOnly: workspaceReadOnly, + onInput: (v) => (workspaceDraft = v) + })} + + + {@render field({ + value: userDraft, + readOnly: false, + onInput: (v) => (userDraft = v) + })} + +
    + {/snippet} +
    +
    diff --git a/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte b/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte new file mode 100644 index 0000000000..c8d360e0f9 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte @@ -0,0 +1,640 @@ + + + + + + + + +{#snippet listPage()} +
    +
    + {#snippet action()} + + {/snippet} + + {#if forkPending} + + Connections are read-only until the first message creates this session's fork. Connecting + or selecting one now would apply to the parent workspace and stop applying once the fork + is created. + + {/if} + {#if loading} +
    + {:else if loadError} +
    + Failed to load MCP connections: {loadError} +
    + {:else if servers.length === 0} + + {:else} +
    + {#each servers as server (server.path)} + {#snippet icon()} + + {/snippet} + {#snippet title()} + {server.path} + {/snippet} + {#snippet subtitle()}{server.description}{/snippet} + {#snippet trailing()} + await toggle(server.path, e.detail)} + /> + openServer(server) + }, + { + displayName: 'Delete', + icon: Trash2, + type: 'delete', + disabled: forkPending, + action: () => (pendingDelete = server.path) + } + ]} + /> + {/snippet} + openServer(server)} + /> + {/each} +
    + {/if} +
    +
    +{/snippet} + +{#snippet detailPage()} +
    + +
    + +
    + + +
    + {#snippet action()} +
    + +
    + {/snippet} + + +
    (editingTouched = true)}> + {#key detailSeq} + {#if detailSeq > 0} + + + {/if} + {/key} +
    +
    +
    +{/snippet} + +{#snippet connectPage()} + +
    + +
    + +
    +
    + {#key connectSeq} + { + // Connecting one is the act of choosing it, and it is keyed on where + // it was created rather than on what is on screen now: a switch + // during the popup would otherwise enable the path in a workspace + // that has no such connection. + if (!setMcpEnabled(connectedWs, path, true)) { + sendUserToast(`Connected ${path}, but could not turn it on. Toggle it here.`, true) + } + closeConnect() + await refresh() + }} + /> + {/key} +
    +
    +{/snippet} + + { + if (pendingDelete) void deleteConnection(pendingDelete) + }} + onCanceled={() => (pendingDelete = undefined)} +> + + This deletes the resource at {pendingDelete}, so the chat and + any flow pointing at it lose the server. Its token variable is kept. To stop this chat from + using the server without deleting it, turn its switch off instead. + + diff --git a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte index 787d9ef4fc..9698180184 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte @@ -6,19 +6,41 @@ import { thinkingPreferences } from './thinkingPreferences.svelte' import CodeDisplay from './script/CodeDisplay.svelte' import LinkRenderer from './LinkRenderer.svelte' - import { workspaceStore } from '$lib/stores' import { extractCandidatePaths, remarkWindmillPaths, workspaceItemRegistry } from './workspaceItems.svelte' import { markdownProse } from '$lib/components/markdownProse' + import DisplayResult from '$lib/components/DisplayResult.svelte' + import { Bot, ExternalLink } from 'lucide-svelte' + import CopyButton from '$lib/components/common/button/CopyButton.svelte' + import { base } from '$lib/base' + import { displayDate } from '$lib/utils' interface Props { message: DisplayMessage + // Workspace the message's paths are resolved against: the one the chat + // operates on, which is not always the one being navigated. + workspace: string | undefined } - let { message }: Props = $props() + let { message, workspace }: Props = $props() + + // The run this answer came out of. Only a flow chat has one — a copilot turn runs in + // the browser — so the job link is absent rather than empty elsewhere. + const jobId = $derived(message.role === 'assistant' ? message.jobId : undefined) + const createdAt = $derived(message.role === 'assistant' ? message.createdAt : undefined) + const runHref = $derived(jobId ? `${base}/run/${jobId}?workspace=${workspace}` : undefined) + // Today's answers show the time alone; the day earns its place only on a conversation + // read back later. Resolved at render, so a chat left open across midnight keeps + // yesterday's format until it is reopened. + const timestamp = $derived.by(() => { + if (!createdAt) return undefined + const at = new Date(createdAt) + const today = new Date().toDateString() === at.toDateString() + return displayDate(at, false, !today) + }) const reasoning = $derived( message.role === 'assistant' ? message.reasoning?.trim() || undefined : undefined @@ -58,6 +80,20 @@ return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s` } + const stepName = $derived(message.role === 'assistant' ? message.stepName : undefined) + + // A flow step can return a file rather than text; the raw JSON would be + // unreadable, so hand it to the result viewer instead of the markdown renderer. + const s3Object = $derived.by(() => { + if (!message.content.startsWith('{')) return undefined + try { + const parsed = JSON.parse(message.content) + return parsed?.type === 'windmill_s3_object' && parsed?.s3 ? parsed : undefined + } catch { + return undefined + } + }) + const candidatePaths = $derived(extractCandidatePaths(message.content)) const rendererPlugin = { renderer: { @@ -69,12 +105,11 @@ // Only populate the registry for messages that contain path-shaped tokens. The // registry still dedups concurrent calls across messages and workspaces. $effect(() => { - const ws = $workspaceStore - if (ws && candidatePaths.length > 0) workspaceItemRegistry.ensureLoaded(ws) + if (workspace && candidatePaths.length > 0) workspaceItemRegistry.ensureLoaded(workspace) }) const plugins = $derived.by(() => { - const ws = $workspaceStore ?? '' + const ws = workspace ?? '' if (!ws || candidatePaths.length === 0) { return [gfmPlugin(), rendererPlugin] } @@ -97,6 +132,19 @@ }) + +{#if stepName} +
    + + {stepName} +
    +{/if} + {#if reasoning} {/if} -{#if message.content} +{#if s3Object} + +{:else if message.content}
    {/if} + +{#if message.content} + +
    + + {#if timestamp} + {timestamp} + {/if} + {#if runHref} + + job {jobId?.slice(0, 8)} + + + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte b/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte new file mode 100644 index 0000000000..c9def4f9e6 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte @@ -0,0 +1,240 @@ + + + + + + + + + + +
    + {/snippet} + + + + + {#if forkPending} + + Skills are read-only until the first message creates this session's fork. Editing one or + turning it off now would apply to the parent workspace and stop applying once the fork is + created. + + {/if} + {#if listNotice} + + {listNotice} + + {/if} + {#if loading} +
    Loading skills…
    + {:else if loadError} +
    + Failed to load skills: {loadError} +
    + {:else if skills.length === 0} + + {:else} +
    + {#if grouped} + {#each tree as node (node.path)} + {@render folder(node, true)} + {/each} + {:else} + {#each skills as skill (skill.path)} + {@render row(skill)} + {/each} + {/if} +
    + {/if} + +
    +{/snippet} + + +{#snippet row(skill: Row)} + {#snippet icon()} + + {/snippet} + {#snippet title()} + + + {!grouped && ambiguous.has(skill.name) ? skill.path : skill.name} + + {/snippet} + {#snippet subtitle()}{skill.description}{/snippet} + {#snippet trailing()} + await toggle(skill.path, e.detail)} + /> + openSkill(skill) + }, + { + displayName: 'Delete', + icon: Trash2, + type: 'delete', + disabled: !skill.canWrite || forkPending, + action: () => (toDelete = skill) + } + ]} + /> + {/snippet} + highlight.hovered(entryIndexByKey.get(skillKey(skill.path)) ?? -1)} + subtitle={skill.description ? subtitle : undefined} + onClick={() => openSkill(skill)} + /> +{/snippet} + + +{#snippet folder(node: SkillTreeNode, root: boolean)} +
    + + +
    highlight.hovered(entryIndexByKey.get(folderKey(node.path)) ?? -1)} + > + + await toggleFolder(node, e.detail)} + /> +
    + {#if !collapsed[node.path]} +
    + {#each node.children as child (child.path)} + {@render folder(child, false)} + {/each} + {#each node.skills as skill (skill.path)} + {@render row(skill)} + {/each} +
    + {/if} +
    +{/snippet} + +{#snippet editorPage()} + + +
    + +
    + +
    + +
    + {#snippet action()} + {#if editing} + +
    + + {#snippet children({ item })} + + + {/snippet} + +
    + {/if} + {/snippet} +
    + +
    + {#if parsed.description} +

    {parsed.description}

    + {/if} +
    + +
    +
    +
    + + +
    + +
    +
    + {contentError ?? ''} +
    + + +
    +
    +
    +
    +
    +
    +{/snippet} + + { + const skill = toDelete + toDelete = undefined + if (skill) await remove(skill) + }} + onCanceled={() => (toDelete = undefined)} +> + + This deletes the resource at {toDelete?.path}, so everyone + who can read it loses the skill. + + + + { + const toImport = [ + ...pendingNew.map((skill) => ({ skill, overwrite: false })), + ...pendingConflicts + .filter((s) => overwriteChoices[s.name]) + .map((skill) => ({ skill, overwrite: true })) + ] + const skipped = pendingSkipped + pendingImport = undefined + pendingSkipped = [] + overwriteChoices = {} + if (toImport.length) await importSkills(toImport, skipped) + else sendUserToast('No skills imported.') + }} + onCanceled={() => { + pendingImport = undefined + pendingSkipped = [] + overwriteChoices = {} + }} +> +
    + + Skills are added under {defaultOwner()}. Move one to a shared + folder from the resources page to share it. + + {#if pendingNew.length} +
    + Add {pendingNew.length} new skill(s): + {pendingNew.map((s) => s.name).join(', ')} +
    + {/if} + {#if pendingConflicts.length} +
    + + {pendingConflicts.length} skill(s) already exist — choose which to overwrite: + +
    + {#each pendingConflicts as conflict (conflict.name)} +
    + {conflict.name} + +
    + {/each} +
    +
    + {/if} + {#if pendingSkipped.length} + {pendingSkipped.length} file(s) will be skipped. + {/if} +
    +
    diff --git a/frontend/src/lib/components/copilot/chat/AssistantToolsSection.svelte b/frontend/src/lib/components/copilot/chat/AssistantToolsSection.svelte new file mode 100644 index 0000000000..99d2d3d286 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/AssistantToolsSection.svelte @@ -0,0 +1,212 @@ + + + + + + t.name} /> + t.description} /> + + + + +{#snippet listPage()} + + +
    +
    + +
    + +
    + {#if rows.length === 0} +
    + {tools.length === 0 ? 'This chat carries no tools.' : 'No tool matches this search.'} +
    + {:else} + +
    + {#each rows as row, index (row.tool.name)} + {#snippet title()} + + {#if row.name}{@html row.name}{:else}{row.tool.name}{/if} + + {/snippet} + {#snippet subtitle()} + {#if row.description}{@html row.description}{:else}{row.tool.description}{/if} + {/snippet} + highlight.hovered(index)} + onClick={() => open(row.tool)} + /> + {/each} +
    + {/if} +
    +
    +{/snippet} + +{#snippet detailPage()} +
    + +
    + +
    + +
    + {#if selected?.description} + +
    {selected.description}
    + {/if} + +
    +
    +{/snippet} diff --git a/frontend/src/lib/components/copilot/chat/ChatCollapsibleCard.svelte b/frontend/src/lib/components/copilot/chat/ChatCollapsibleCard.svelte index f92e347da9..f9a5950e1e 100644 --- a/frontend/src/lib/components/copilot/chat/ChatCollapsibleCard.svelte +++ b/frontend/src/lib/components/copilot/chat/ChatCollapsibleCard.svelte @@ -6,6 +6,11 @@ interface Props { label: string + /** Set before the label and left unemphasised, so `labelClass` lifts the subject alone: + * the part of the heading that is grammar stays quiet. A step lighter than the row's own + * label weight, because a caller emphasising the label is also setting a proportional + * typeface, whose medium reads heavier than the mono one at this size. */ + labelPrefix?: string expanded: boolean onToggle: () => void // A card with nothing to reveal keeps the header inert (no chevron, no @@ -13,6 +18,9 @@ toggleable?: boolean // Sweeps a highlight across the label while the row is in progress. shimmer?: boolean + // Ahead of the label, inside the toggle button: a status that reads as part of the + // row rather than as another control, leaving the chevron next to the label it opens. + headerLeft?: Snippet // Pinned to the right of the header row, outside the toggle button. headerRight?: Snippet // Always-visible content between the header and the expandable body. @@ -26,10 +34,12 @@ let { label, + labelPrefix, expanded, onToggle, toggleable = true, shimmer = false, + headerLeft, headerRight, belowHeader, children, @@ -49,7 +59,8 @@ highlight && 'text-emphasis' )} > - {label} + {#if labelPrefix}{labelPrefix} {/if}{label} {/snippet} @@ -61,7 +72,9 @@ )} onclick={onToggle} disabled={!toggleable} + aria-expanded={toggleable ? expanded : undefined} > + {@render headerLeft?.()} {#if shimmer} {@render labelText(false)} diff --git a/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte b/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte index 995a84708c..5c507deb33 100644 --- a/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte +++ b/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte @@ -2,6 +2,7 @@ import DrillPicker from '$lib/components/DrillPicker.svelte' import type { DrillLeaf, DrillNode } from '$lib/components/drillPicker' import type { ChatCommandItem } from './global/core' + import { ambiguousSkillNames } from './skills/skillResources' interface Props { skills: ChatCommandItem[] @@ -24,15 +25,22 @@ skill: 'Skills' } - // No `secondary`: rows show just the command; the full description lives in - // the hover tooltip (rowTooltip below). It stays in `searchableText` so - // filtering by description keeps working. + // Two folders can each hold a skill of the same name, and `/name` cannot then + // say which one is meant. Those rows show their path so the two are at least + // distinguishable; unambiguous rows stay bare, with the description in the + // hover tooltip (rowTooltip below) and in `searchableText` so filtering by it + // keeps working. + const ambiguous = $derived(ambiguousSkillNames(skills.filter((s) => s.path !== undefined))) + const tree = $derived[]>( skills.map((skill) => ({ type: 'leaf' as const, - key: `skill:${skill.name}`, + // Keyed by path where there is one: names are not unique across folders, + // and a duplicate key breaks the keyed list and its ambiguous-resolve nav. + key: `skill:${skill.path ?? skill.name}`, label: `/${skill.name}`, - searchableText: `${skill.name} ${skill.description}`, + secondary: ambiguous.has(skill.name) ? skill.path : undefined, + searchableText: `${skill.name} ${skill.path ?? ''} ${skill.description}`, section: skill.kind ? SECTION_LABELS[skill.kind] : undefined, data: skill })) diff --git a/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte b/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte index d2cf1dfee6..68a65d75fc 100644 --- a/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte +++ b/frontend/src/lib/components/copilot/chat/ChatContextPicker.svelte @@ -19,7 +19,6 @@ On a workspace-leaf pick, emits a reference-only `WorkspaceScriptElement` / Content is materialized at message-prep time by `AIChatManager` — see PR #9216. -->
    -
    -
    + -
    -
    + {#if expanded}
    !isMentionContext(c)) + } + getAvailableContext() { return this.availableContext } diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 2502cd075b..976b969b3f 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -1,4 +1,5 @@ - -
    +
    {@render leading?.()} @@ -814,10 +817,7 @@ {placeholder} class={twMerge( 'textarea-input resize-none caret-black dark:caret-white overflow-clip', - // The box (border/ring) lives on the wrapper; kill the textarea's own - // @tailwindcss/forms border, focus ring, and background so only the - // wrapper reads as the field. - '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0', + COMPOSER_FIELD_RESET, CHAT_INPUT_PADDING, className )} diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index ffa06ca5df..b49bc582ef 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -1,36 +1,48 @@ {#if visible} - - -
    -
    -
    -
    -
    - {#snippet text()} + + + {#snippet tooltip()}

    Context usage

    @@ -85,5 +83,5 @@ {/if}

    {/snippet} -
    + {/if} diff --git a/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte b/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte index 65abb0e1c5..6b4cbbd710 100644 --- a/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte +++ b/frontend/src/lib/components/copilot/chat/DatatableCreationPolicy.svelte @@ -5,11 +5,13 @@ const aiChatManager = getAiChatManager() import DefaultDatabaseSelector from '$lib/components/raw_apps/DefaultDatabaseSelector.svelte' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() import { createDatatablesResource } from '$lib/components/raw_apps/datatableUtils.svelte' // Load available datatables from workspace using shared utility - const datatables = createDatatablesResource(() => $workspaceStore) + const datatables = createDatatablesResource(() => $operatingWorkspace) const hasNoDatatables = $derived((datatables.current?.length ?? 0) === 0) @@ -67,6 +69,7 @@ diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 24d7284b95..7843169dd9 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -1,9 +1,15 @@ -import { type DBSchema as IDBSchema, type IDBPDatabase } from 'idb' +import { + type DBSchema as IDBSchema, + type IDBPDatabase, + type IDBPTransaction, + type StoreNames +} from 'idb' import type { ChatJob, DisplayMessage } from './shared' import { expanded, messageDraft } from './chatDraft' import { createLongHash } from '$lib/editorLangUtils' import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb' -import { scopedKey } from '$lib/userScopedStorage' +import { emailOfScopedKey, scopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import type { PersistedContextUsage } from './tokenUsage' import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils' @@ -14,8 +20,9 @@ import { randomUUID } from '$lib/utils/uuid' // shared browser. The bare name is also the legacy (pre-namespacing) DB, claimed // once on first login. const DB_NAME = 'copilot-chat-history' -// v3 adds the images blob store (replacing v2's short-lived toolImages store). -const DB_VERSION = 3 +// v3 adds the images blob store (replacing v2's short-lived toolImages store); v4 indexes +// chats by session. +const DB_VERSION = 4 /** Newest image blobs kept per chat; each is a bounded (≤1568px) data URL. */ const MAX_IMAGES_PER_CHAT = 30 /** Marks a persisted image whose bytes live in the `images` store. */ @@ -45,6 +52,7 @@ interface ChatSchema extends IDBSchema { // chats predating this feature. Persisted out-of-band like modifiedItems. backgroundJobs?: ChatJob[] } + indexes: { 'by-session': string } } // Image bytes, out-of-band from the chat record on purpose: the record is // re-cloned into IndexedDB on every saveChat, while a blob is written once @@ -64,9 +72,19 @@ interface ChatSchema extends IDBSchema { } } -function createChatStore(db: IDBPDatabase): void { - if (!db.objectStoreNames.contains('chats')) { - db.createObjectStore('chats', { keyPath: 'id' }) +/** A persisted chat, exactly as the store holds it (image refs, not bytes). */ +export type StoredChat = ChatSchema['chats']['value'] + +function createChatStore( + db: IDBPDatabase, + tx: IDBPTransaction[], 'versionchange'> +): void { + const chats = db.objectStoreNames.contains('chats') + ? tx.objectStore('chats') + : db.createObjectStore('chats', { keyPath: 'id' }) + // Lets the session backup find a session's chats without reading every record. + if (!chats.indexNames.contains('by-session')) { + chats.createIndex('by-session', 'sessionId') } // v2 briefly kept full-resolution tool screenshots in their own store; the // general blob store below covers them now. @@ -117,7 +135,9 @@ async function claimLegacyChatDb( { openDB, deleteDB }: UserScopedDbMigrateDeps ): Promise { if ((await scopedDb.count('chats')) > 0) return - const legacy = await openDB(DB_NAME, 1, { upgrade: createChatStore }) + const legacy = await openDB(DB_NAME, 1, { + upgrade: (db, _oldVersion, _newVersion, tx) => createChatStore(db, tx) + }) const legacyChats = await legacy.getAll('chats') if (legacyChats.length > 0) { const tx = scopedDb.transaction('chats', 'readwrite') @@ -156,6 +176,136 @@ export async function readChatModifiedItems(chatId: string): Promise(DB_NAME, { + version: DB_VERSION, + upgrade: createChatStore, + migrate: migrateLegacyChatDb +}) + +async function backupDb(email: string): Promise | undefined> { + const db = await backupDbh.whenReady() + return db && db.name === scopedKeyFor(DB_NAME, email) ? db : undefined +} + +/** Test-only: let go of the backup's handle so the next call opens the test's fresh + * IndexedDB rather than the connection a previous test left. */ +export function __resetBackupStoreForTesting(): void { + backupDbh.close() +} + +/** Ids of the chats tagged with this session, or undefined when the store is unavailable. */ +export async function listSessionChatIds( + sessionId: string, + email: string +): Promise { + const db = await backupDb(email) + if (!db) return undefined + return (await db.getAllKeysFromIndex('chats', 'by-session', sessionId)).map(String) +} + +export async function readStoredChat(id: string, email: string): Promise { + const db = await backupDb(email) + return db?.get('chats', id) +} + +/** Ids of the image blobs a chat owns, or undefined when the store is unavailable. */ +export async function listChatImageIds( + chatId: string, + email: string +): Promise { + const db = await backupDb(email) + if (!db) return undefined + return (await imageKeysForChat(db, chatId)).map(String) +} + +export async function readImageDataUrl(id: string, email: string): Promise { + const db = await backupDb(email) + return (await db?.get('images', id))?.dataUrl +} + +export interface RestoredImage { + id: string + chatId: string + dataUrl: string +} + +/** + * Write restored chats and image blobs, leaving any that already exist alone: a record + * this browser wrote since is newer than the backup it came from. False when the store + * could not be reached, which the caller must not record as a restore. + */ +export async function importStoredChats( + chats: StoredChat[], + images: RestoredImage[], + email: string, + overwrite = false +): Promise { + const db = await backupDb(email) + if (!db) return false + const tx = db.transaction(['chats', 'images'], 'readwrite') + const chatStore = tx.objectStore('chats') + const imageStore = tx.objectStore('images') + const savedAt = Date.now() + for (const image of images) { + if (overwrite || (await imageStore.getKey(image.id)) === undefined) { + await imageStore.put({ id: image.id, chatId: image.chatId, dataUrl: image.dataUrl, savedAt }) + } + } + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const chat of chats) { + const existing = await chatStore.get(chat.id) + if (existing === undefined || (overwrite && existing.lastModified <= chat.lastModified)) { + await chatStore.put(chat) + } + } + await tx.done + return true +} + +/** Every chat tagged with the session, and their images: for a session past its workspace's + * retention, which no runtime has mounted. */ +export function deleteSessionChats(sessionId: string, email: string): Promise { + return pruneSessionChats(sessionId, undefined, new Set(), email) +} + +/** Deletes chats of the session (with their images) and these images: what an earlier restore + * staged for it and the backup no longer has. `chats` names the ones to go; undefined is every + * chat of the session. False when nothing could be deleted. */ +export async function pruneSessionChats( + sessionId: string, + chats: Set | undefined, + images: Set, + email: string +): Promise { + if (chats?.size === 0 && images.size === 0) return true + const db = await backupDb(email) + if (!db) return false + try { + const tx = db.transaction(['chats', 'images'], 'readwrite') + const chatStore = tx.objectStore('chats') + const imageStore = tx.objectStore('images') + for (const chatId of await chatStore.index('by-session').getAllKeys(sessionId)) { + if (chats && !chats.has(String(chatId))) continue + await chatStore.delete(chatId) + const keys = await imageStore + .index('by-chat') + .getAllKeys(IDBKeyRange.bound([chatId, -Infinity], [chatId, Infinity])) + for (const key of keys) await imageStore.delete(key) + } + for (const id of images) await imageStore.delete(id) + await tx.done + return true + } catch (err) { + console.error('Could not prune chats for session', err) + return false + } +} + export default class HistoryManager { // Per-instance handle to the shared per-user DB lifecycle. There is one // HistoryManager per AIChatManager (the singleton + one per session runtime), @@ -244,7 +394,10 @@ export default class HistoryManager { .sort((a, b) => b.lastModified - a.lastModified) ) - async init() { + /** With `session`, loads only that session's chats plus `session.chatId`, which + * may predate session tagging. Loading every chat instead costs one full copy + * of the user's history per session runtime. */ + async init(session?: { id: string; chatId?: string }) { // (Re)initializing adopts a new identity's history: drop the previous // identity's cached blob ids with it. this.imageIdByUrl.clear() @@ -254,7 +407,16 @@ export default class HistoryManager { const db = await this.dbh.whenReady() if (!db) return try { - const chats = await db.getAll('chats') + let chats: StoredChat[] + if (session) { + chats = await db.getAllFromIndex('chats', 'by-session', session.id) + if (session.chatId && !chats.some((c) => c.id === session.chatId)) { + const current = await db.get('chats', session.chatId) + if (current) chats.push(current) + } + } else { + chats = await db.getAll('chats') + } this.savedChats = chats.reduce( (acc, chat) => { acc[chat.id] = chat @@ -289,7 +451,10 @@ export default class HistoryManager { const snapshot = $state.snapshot(existing) const updated = { ...snapshot, sessionId } this.savedChats = { ...this.savedChats, [chatId]: updated } - await this.enqueueDbWrite((db) => db.put('chats', updated)) + await this.enqueueDbWrite(async (db) => { + await db.put('chats', updated) + markSessionDirty(sessionId, chatId, emailOfScopedKey(DB_NAME, db.name)) + }) } getPastChats() { @@ -565,6 +730,13 @@ export default class HistoryManager { const keep = this.keptImageIds(refs) await this.writeKeptImageBlobs(db, updatedChat.id, blobs, keep) await db.put('chats', updatedChat) + if (updatedChat.sessionId) { + markSessionDirty( + updatedChat.sessionId, + updatedChat.id, + emailOfScopedKey(DB_NAME, db.name) + ) + } // Best-effort: the record is already committed, so a failed cleanup // (e.g. a user switch closed this handle mid-op) must not turn a // successful save into a rejection — the orphans are reclaimed by @@ -589,6 +761,7 @@ export default class HistoryManager { } deletePastChat(id: string) { + const sessionId = this.savedChats[id]?.sessionId this.savedChats = Object.fromEntries( Object.entries(this.savedChats).filter(([key]) => key !== id) ) @@ -596,9 +769,39 @@ export default class HistoryManager { await db.delete('chats', id) const keys = await imageKeysForChat(db, id) await Promise.all(keys.map((key) => db.delete('images', key))) + if (sessionId) markSessionDirty(sessionId, id, emailOfScopedKey(DB_NAME, db.name)) }).catch((err) => console.error('Could not delete chat', err)) } + /** Re-read one chat from the store into the in-memory mirror, for a record + * another tab wrote after this manager last read it. `init()` is the wrong + * tool: it re-reads the user's entire history to pick up a single chat. + * + * 'missing' is a fact about the conversation (the store holds nothing under + * this id); 'unavailable' is a fact about this browser. Callers act on the + * first and must not act on the second — treating a closed database as an + * empty chat would throw away a transcript that is merely unreadable. */ + async reloadChat(id: string): Promise<'loaded' | 'missing' | 'unavailable'> { + const db = await this.dbh.whenReady() + if (!db) return 'unavailable' + try { + const chat = await db.get('chats', id) + if (!chat) { + // Drop the mirror too. `loadPastChat` reads from it and never from the + // store, so a copy left behind here is a deleted chat that comes back + // on the next rotation onto this id. + const { [id]: _gone, ...rest } = this.savedChats + this.savedChats = rest + return 'missing' + } + this.savedChats = { ...this.savedChats, [id]: chat } + return 'loaded' + } catch (err) { + console.error('Could not reload chat', err) + return 'unavailable' + } + } + async loadPastChat(id: string) { const chat = this.savedChats[id] if (!chat) return diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index ffcc065157..3ee9edad39 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -757,3 +757,95 @@ describe('HistoryManager modified-items mask persistence', () => { expect(hm.getModifiedItems(id)).toBeUndefined() }) }) + +describe('HistoryManager.reloadChat', () => { + it('picks up another tab’s write, and tells an empty chat from an unreadable store', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [{ role: 'user', content: 'before the other tab ran' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + // The other tab's turn, written straight to the store this one shares. + const db = await openDB('copilot-chat-history::admin@test') + const row = (await db.get('chats' as never, chatId)) as any + row.displayMessages = [{ role: 'user', content: 'written by the driving tab' }] + await db.put('chats' as never, row) + db.close() + + expect(await hm.reloadChat(chatId)).toBe('loaded') + const chat = await hm.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).content).toBe('written by the driving tab') + + // A chat the store does not hold — distinct from 'unavailable' below: + // 'missing' evicts the in-memory mirror, so conflating the two would let + // a store that merely failed to open erase transcripts this tab holds. + expect(await hm.reloadChat('no-such-chat')).toBe('missing') + }) + + it('evicts the mirrored copy of a chat the driver deleted', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [{ role: 'user', content: 'deleted by the driving tab' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + const db = await openDB('copilot-chat-history::admin@test') + await db.delete('chats' as never, chatId) + db.close() + + expect(await hm.reloadChat(chatId)).toBe('missing') + // loadPastChat serves the mirror, so a copy left behind would resurrect the + // deleted transcript the next time this id came round again. + expect(await hm.loadPastChat(chatId)).toBeUndefined() + }) + + it('reports a store it cannot open as unavailable, never as missing', async () => { + ;(globalThis as any).indexedDB = { + open: () => { + throw new Error('blocked') + } + } + const hm = new HistoryManager() + await hm.init() + + expect(await hm.reloadChat(hm.getCurrentChatId())).toBe('unavailable') + }) +}) + +describe('HistoryManager.init for a session', () => { + it("loads the session's chats and its current chat, not other sessions'", async () => { + const setup = new HistoryManager() + await setup.init() + setup.close() + const chat = (id: string, sessionId?: string) => ({ + id, + actualMessages: [], + displayMessages: [], + title: id, + lastModified: 1, + ...(sessionId ? { sessionId } : {}) + }) + const db = await openDB('copilot-chat-history::admin@test') + await db.put('chats' as never, chat('mine', 's1') as never) + await db.put('chats' as never, chat('other-session', 's2') as never) + await db.put('chats' as never, chat('untagged-current') as never) + await db.put('chats' as never, chat('untagged-other') as never) + db.close() + + const hm = new HistoryManager() + await hm.init({ id: 's1', chatId: 'untagged-current' }) + + // The current chat can predate session tagging, so it is read by key. + expect( + hm + .getAllSavedChats() + .map((c) => c.id) + .sort() + ).toEqual(['mine', 'untagged-current']) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index 5d79251fd2..adc3fe6184 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -3,12 +3,17 @@ import { ExternalLink, PanelRight } from 'lucide-svelte' import { Button } from '$lib/components/common' import RowIcon from '$lib/components/common/table/RowIcon.svelte' - import { runToolDisplayAction } from './createdResourceActions.svelte' + import { newTabModifier } from '$lib/attachments/newTabModifier.svelte' + import { + hasToolDisplayActionHandler, + runToolDisplayAction + } from './createdResourceActions.svelte' import { workspaceItemAction, type WindmillItemKind, type WorkspaceItemTargetKind } from './workspaceItems.svelte' + import { safeHref } from './safeHref' type Props = { href?: string @@ -16,6 +21,7 @@ 'data-wm-kind'?: WindmillItemKind 'data-wm-path'?: string 'data-wm-target-kind'?: WorkspaceItemTargetKind + 'data-wm-raw-app'?: string title?: string } let { @@ -24,10 +30,29 @@ 'data-wm-kind': wmKind, 'data-wm-path': wmPath, 'data-wm-target-kind': wmTargetKind, + 'data-wm-raw-app': wmRawApp, title }: Props = $props() - const drawerAction = $derived(workspaceItemAction(wmKind, wmPath, wmTargetKind)) + // The drawers ride with the docked chat and the session tabs with the sessions page, so a + // surface can render this pill with nothing able to open one. + const available = $derived.by(() => { + const action = workspaceItemAction(wmKind, wmPath, wmTargetKind, wmRawApp === 'true') + return action && hasToolDisplayActionHandler(action.type) ? action : undefined + }) + // Only the preview panel takes the plain click. A drawer keeps its own button beside an + // outbound link: the docked chat mounts drawer handlers on nearly every page, so claiming + // that click would redirect these pills far outside the sessions page. + const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) + const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + + const allowedHref = $derived(safeHref(href, window.location.href)) + + const modifier = newTabModifier() + + const hint = $derived( + previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab` + ) async function openDrawer(event?: Event) { event?.preventDefault() @@ -36,27 +61,50 @@ await runToolDisplayAction(drawerAction) } } + + async function onclick(event: MouseEvent) { + // Modifier clicks are the only remaining route to the tab once the plain click is + // spoken for, so leave them to the browser. + if (!previewAction || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) return + event.preventDefault() + await runToolDisplayAction(previewAction) + } -{#if href} +{#if allowedHref} {#if wmKind} - + + - - + + + + + + + + {#if previewAction && !modifier.held} + + {:else} + + {/if} + {@render children?.()} - - - {#if drawerAction}
    - {/each} -
    - {/if} -
    - - { - if (pendingDisconnect) void disconnect(pendingDisconnect) - }} - onCanceled={() => (pendingDisconnect = undefined)} - > - - This deletes the resource at {pendingDisconnect}, so the chat - and any flow pointing at it lose the server. Its token variable is kept. - - - - diff --git a/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte b/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte index f5fdc7a5d3..b80fa17bb1 100644 --- a/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte +++ b/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte @@ -3,7 +3,7 @@ import { FileText, X } from 'lucide-svelte' import ContextElementBadge from './ContextElementBadge.svelte' import { contextElementKey } from './context' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' // The single message typed while a turn was streaming, waiting to be // auto-sent when the turn finishes. Rendered above the whole input stack @@ -11,7 +11,7 @@ // conversation". Pressing Enter again appends another line to it; clicking // the chip body (its X, or ArrowUp in the empty input) removes it and // restores its content into the input so nothing is lost. - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() -{#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0 || (aiChatManager.queuedContext?.length ?? 0) > 0} +{#if chatHost.queuedMessage || chatHost.queuedImages.length > 0 || chatHost.queuedFiles.length > 0 || chatHost.queuedBlobs.length > 0 || (chatHost.queuedContext?.length ?? 0) > 0}
    - {#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0} + {#if chatHost.queuedMessage || chatHost.queuedImages.length > 0 || chatHost.queuedFiles.length > 0 || chatHost.queuedBlobs.length > 0} - {:else if aiChatManager.queuedContext?.length} + {:else if chatHost.queuedContext?.length}
    - {#each aiChatManager.queuedContext as element (contextElementKey(element))} + {#each chatHost.queuedContext as element (contextElementKey(element))} {/each}
    @@ -82,7 +97,7 @@ iconOnly title="Remove queued message and put it back in the input" startIcon={{ icon: X }} - on:click={() => aiChatManager.dequeueMessage()} + on:click={() => chatHost.dequeueMessage()} />
    {/if} diff --git a/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte b/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte new file mode 100644 index 0000000000..6072b6e341 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte @@ -0,0 +1,242 @@ + + + + +
    + +
    + +
    +
    + {#if hasArgs} + + + {:else} +

    This {runnableKind} takes no arguments.

    + {/if} +
    +
    + + {#if fades.bottom} +
    + {/if} +
    + + +
    + {#if runForm.clearedKeys?.length} +

    + Sent in a shape this field has no reading of, so it opened empty: + {runForm.clearedKeys.join(', ')} +

    + {/if} + {#if runForm.resetKeys?.length} +

    + Disabled by this {runnableKind}, so it will run with its default: + {runForm.resetKeys.join(', ')} +

    + {/if} + {#if runForm.strippedKeys?.length} +

    + A file, so it opened empty for you to attach: + {runForm.strippedKeys.join(', ')} +

    + {/if} + {#if planMode} +

    {PLAN_MODE_MESSAGES.runFormRefused}

    + {/if} + + +
    + + +
    +
    +
    diff --git a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte new file mode 100644 index 0000000000..4aff51d002 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte @@ -0,0 +1,687 @@ + + + +{#snippet status()} + + {#if !pending && !inspected} + + {statusTime} + + {/if} +{/snippet} + + +{#snippet previewChip()} + +{/snippet} + + + + (toggled = { id: message.tool_call_id, open: !expanded })} + headerLeft={status} + headerRight={previewTarget ? previewChip : undefined} + class="scroll-mb-8" + labelClass="font-main font-medium text-primary" + contentClass="p-0 overflow-hidden" +> + {#if formInPreview} +
    + These inputs are open in the preview panel. +
    + {:else if pending && runForm} + + {:else} + +
    + + (userTab = { id: message.tool_call_id, value: e.detail })} + class="h-8 px-3 font-main" + wrapperClass="shrink-0" + slidingIndicator + > + {#if !rawView && !jobPending} + {#each tabs as tab (tab.value)} + + + + {#snippet extra()} + {#if tab.value === 'logs' && logLineCount > 0} + {logLineCount} + {/if} + {/snippet} + + + {/each} + {/if} +
    + (jsonView = { id: message.tool_call_id, on: e.detail })} + size="2xs" + options={{ right: 'JSON', rightTooltip: 'Show this call as raw JSON' }} + lightMode + /> +
    +
    + + +
    + +
    + {#if rawView} +
    + + + + +
    + {:else if jobPending} + +
    + {#if fetchFailed === message.tool_call_id} + This run could not be read. It may have been deleted, or be in another workspace. + Its result is on the JSON toggle. + {:else} + + + Loading this run... + + {/if} +
    + {:else} + + {#key activeTab} +
    + {#if activeTab === 'input'} + + + {:else if activeTab === 'logs'} + {#if logs.trim()} + {#if logs.length >= MAX_LOG_LENGTH} +

    + Tail of the logs, the last {MAX_LOG_LENGTH} characters. +

    + {/if} +
    {logs}
    + {:else} +

    + {inspectedJob?.logsFailed + ? 'Logs could not be read.' + : running + ? 'No logs yet.' + : 'No logs.'} +

    + {/if} + {#if running && !inspected} +
    + + streaming +
    + {/if} + + {:else if !inspected && failed} +
    {message.error}
    + {:else if streaming} + + + {:else if resultValue !== undefined} + + + {:else if canceled} + +
    + +

    {cancelReason}

    + {#if !ran} +

    + The inputs it would have run with are on the Inputs tab. +

    + {/if} +
    + {:else} +

    This run returned no result.

    + {/if} +
    + {/key} + {/if} +
    +
    + + + {#if fades.bottom && fadeBody} +
    + {#if activeTab === 'logs'} +
    + {/if} + {/if} +
    + + {#if running && chatJob} + +
    + +
    + {/if} + {/if} +
    diff --git a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte index 699b79a292..fcf943cb34 100644 --- a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte @@ -1,7 +1,7 @@ diff --git a/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte index c4b2d0cc21..9d1bf698f7 100644 --- a/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolContentDisplay.svelte @@ -1,6 +1,7 @@ @@ -136,17 +127,18 @@ {:else if hasContent}
    -
    {formatJson($state.snapshot(content))}
    - {#if showFade && canScrollDown} + {#if showFade && fades.bottom}
    {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 34b1adcf68..5405a6ea25 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -9,8 +9,10 @@ CircleMinus, FileText, PanelRight, - Lock + Lock, + ExternalLink } from 'lucide-svelte' + import { base } from '$lib/base' import { EXIT_PLAN_MODE_TOOL, isPlanCardTool, @@ -21,9 +23,9 @@ } from './planMode' import { Button } from '$lib/components/common' import { markdownProse } from '$lib/components/markdownProse' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() import { isActiveUserQuestion, type ToolDisplayMessage } from './shared' import ChatCollapsibleCard from './ChatCollapsibleCard.svelte' import { twMerge } from 'tailwind-merge' @@ -37,8 +39,11 @@ import ToolMessageActions from './ToolMessageActions.svelte' import ToolPreviewCard from './ToolPreviewCard.svelte' import AskUserQuestionDisplay from './AskUserQuestionDisplay.svelte' + import RunScriptCard from './RunScriptCard.svelte' import WebSearchSourcesDisplay from './WebSearchSourcesDisplay.svelte' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' + import McpServerIcon from '$lib/components/mcp/McpServerIcon.svelte' + import { resolveMcpServerMark } from '$lib/components/mcp/serverMark' interface Props { message: ToolDisplayMessage @@ -46,6 +51,11 @@ let { message }: Props = $props() + // Recorded by the call itself, from the connected-server list rather than from the + // model's arguments — which is what lets a reloaded transcript still resolve it, and + // what keeps a path the model made up from being read as a workspace resource. + const mcpServer = $derived(message.mcpServer) + const isPlanReview = $derived(message.toolName === EXIT_PLAN_MODE_TOOL) const isPlanCard = $derived(isPlanCardTool(message.toolName)) const planCopy = $derived( @@ -61,7 +71,7 @@ const planLabel = $derived((planState && planCopy?.[planState]) ?? '') const planDoc = $derived( message.planArtifactId - ? aiChatManager.artifacts.artifacts.find((a) => a.id === message.planArtifactId) + ? chatHost.artifacts.artifacts.find((a) => a.id === message.planArtifactId) : undefined ) // The version this card wrote, not the document's current one, since later proposals move it on. @@ -117,6 +127,12 @@ isActiveUserQuestion(message) ? message.userQuestion : undefined ) + // The run card owns this call from the form to whatever settled it, cancelling included: + // the card is the call, and a run the user stopped is not a different kind of thing. + // A call that inspected a run rather than starting one gets the same card, bound to + // the job it named — what happened in a run reads the same either way. + const isRunCard = $derived(Boolean(message.runForm || message.inspectedRun)) + // The preview chip sits on the header row (to the right of the tool-call text); // shown once the tool settled, never while loading/erroring/awaiting confirmation. const showPreviewChip = $derived( @@ -140,6 +156,8 @@ {message.toolName} {/if}
    +{:else if isRunCard} + {:else if planState} @@ -187,7 +205,7 @@ title="Open this plan in the side panel: {planDoc.name}" startIcon={{ icon: FileText, classes: PLAN_MODE_TEXT_COLOR }} endIcon={{ icon: PanelRight }} - on:click={() => aiChatManager.openArtifact?.(planDoc.id, planDoc.name, planCardVersion)} + on:click={() => chatHost.openArtifact?.(planDoc.id, planDoc.name, planCardVersion)} > Plan @@ -235,6 +253,30 @@ {/if} {/snippet} + {#snippet jobLink()} + + job {message.jobId?.slice(0, 8)} + + + {/snippet} + + + {#snippet serverMark()} + {#if mcpServer?.workspace && mcpServer.path} + {#await resolveMcpServerMark(mcpServer.workspace, mcpServer.path) then mark} + + {/await} + {/if} + {/snippet} + @@ -250,7 +292,8 @@ headerClass={message.needsConfirmation ? 'opacity-80' : ''} labelClass={showPreviewChip ? 'truncate' : ''} contentClass="space-y-3" - headerRight={showPreviewChip ? previewChip : undefined} + headerRight={showPreviewChip ? previewChip : message.jobId ? jobLink : undefined} + headerLeft={mcpServer?.workspace ? serverMark : undefined} > {#snippet belowHeader()} diff --git a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte index 02612492a5..84ee04d06f 100644 --- a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte @@ -28,7 +28,10 @@ import MqttIcon from '$lib/components/icons/MqttIcon.svelte' import AmqpIcon from '$lib/components/icons/AmqpIcon.svelte' import NatsIcon from '$lib/components/icons/NatsIcon.svelte' - import { runToolDisplayAction } from './createdResourceActions.svelte' + import { + hasToolDisplayActionHandler, + runToolDisplayAction + } from './createdResourceActions.svelte' import type { CreatedResourceTriggerKind, ToolDisplayAction } from './shared' interface Props { @@ -122,17 +125,22 @@
    {card.title}
    {card.subtitle}
    - + + {#if hasToolDisplayActionHandler(action.type)} + + {/if} {/each} diff --git a/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte b/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte index 81d00fd8e4..b5f4fa8bae 100644 --- a/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolPreviewCard.svelte @@ -8,15 +8,26 @@ interface Props { card: { kind: PreviewCardKind; path: string } + /** Opens something other than the item's own preview — the run card opens the call + * it owns, which is a form before it is a run. */ + onOpen?: () => void + title?: string + /** The kind icon says what the chip opens. A card that already names its own runnable + * in the row above has said it, and repeating it there reads as a second subject. */ + kindIcon?: boolean } - let { card }: Props = $props() + let { card, onOpen, title, kindIcon = true }: Props = $props() const kindLabel = $derived(card.kind === 'raw_app' ? 'app' : card.kind) let opening = $state(false) async function open() { if (opening) return + if (onOpen) { + onOpen() + return + } opening = true try { await runToolDisplayAction(openItemPreviewAction(card.kind, card.path)) @@ -30,9 +41,11 @@ variant="default" unifiedSize="2xs" disabled={opening} - title="Open {kindLabel} preview: {card.path}" + title={title ?? `Open ${kindLabel} preview: ${card.path}`} onClick={open} - startIcon={{ icon: RowIcon as unknown as IconType, props: { kind: card.kind, size: 12 } }} + startIcon={kindIcon + ? { icon: RowIcon as unknown as IconType, props: { kind: card.kind, size: 12 } } + : undefined} endIcon={{ icon: PanelRight }} wrapperClasses="shrink-0" > diff --git a/frontend/src/lib/components/copilot/chat/UsageMeter.svelte b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte new file mode 100644 index 0000000000..26e4758953 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte @@ -0,0 +1,39 @@ + + + +
    +
    +
    +
    +
    + {#snippet text()} + {@render tooltip()} + {/snippet} +
    diff --git a/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte b/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte index ff71c44160..075fba6768 100644 --- a/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte @@ -2,6 +2,7 @@ import { Globe } from 'lucide-svelte' import { SvelteSet } from 'svelte/reactivity' import type { WebSearchSource } from './shared' + import { isOfflineReplay } from '$lib/components/recording/offlineReplay.svelte' interface Props { sources: WebSearchSource[] @@ -38,6 +39,12 @@ const failedFavicons = new SvelteSet() + // The favicon is a request to a third party, and the public replay page promises + // to issue none — a recording comes from an arbitrary origin, so its cited + // hostnames must not leak from a viewer's browser either. Degrades to the same + // Globe the blocked/failed case already uses. + const noFavicons = $derived(isOfflineReplay()) + // Favicons come from Google's public favicon service, which discloses each // consulted hostname to a third party from the user's browser — an accepted // tradeoff for now (blocked/air-gapped environments degrade to the Globe @@ -62,7 +69,7 @@ title={source.url} class="flex items-center gap-2 py-1 px-1.5 rounded hover:bg-surface-hover min-w-0" > - {#if failedFavicons.has(hostname)} + {#if noFavicons || failedFavicons.has(hostname)} {:else} ) => + ({ def: { type: 'function', function: { name, description, parameters } } }) as unknown as Tool<{}> + +describe('summarizeTools', () => { + // The modal re-derives this on every turn, so an unsorted list would reshuffle + // under the reader as tools come and go. + it('sorts by name and tolerates a tool with no description', () => { + expect(summarizeTools([tool('run_script', 'Run it.'), tool('deploy')])).toEqual([ + { name: 'deploy', description: '', parameters: { required: [] } }, + { name: 'run_script', description: 'Run it.', parameters: { required: [] } } + ]) + }) + + // `SchemaViewer` renders no argument table at all without `required`, and a tool + // whose arguments are all optional legitimately ships without it. + it('defaults required without dropping the declared schema', () => { + const [summary] = summarizeTools([ + tool('open_page', 'Open it.', { type: 'object', properties: { page: { type: 'string' } } }) + ]) + expect(summary.parameters).toEqual({ + required: [], + type: 'object', + properties: { page: { type: 'string' } } + }) + }) +}) + +describe('countReadyAttachments', () => { + // A folder's own status is an aggregate, and `readyFiles()` filters out the + // placeholder row an empty or all-binary folder keeps — so counting on the folder + // would claim files the assistant cannot open, under a heading about what it can. + it('counts a folder by its readable children, not its own status', () => { + const folders = [ + { files: [{ status: 'indexing' as const }, { status: 'ready' as const }] }, + { files: [] }, + { files: [{ status: 'locked' as const }] } + ] + expect(countReadyAttachments(folders, [])).toBe(1) + }) + + it('counts loose files on their own status', () => { + const files = [ + { status: 'ready' as const }, + { status: 'error' as const }, + { status: 'ready' as const } + ] + expect(countReadyAttachments([], files)).toBe(2) + }) +}) + +describe('attachmentStatusLabel', () => { + // Every unreadable status has to say why: the file tools operate on `readyFiles()`, + // so a row that reads like the usable ones is the one place the panel could claim + // something the assistant cannot open. + it('labels every status the file tools cannot read, and only those', () => { + expect(attachmentStatusLabel('ready')).toBeUndefined() + for (const status of ['locked', 'unavailable', 'indexing', 'error'] as const) { + expect(attachmentStatusLabel(status)).toBeTruthy() + } + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/agentContext.ts b/frontend/src/lib/components/copilot/chat/agentContext.ts new file mode 100644 index 0000000000..99b334bcd5 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/agentContext.ts @@ -0,0 +1,58 @@ +import type { Tool } from './shared' +import type { AttachedFileStatus } from './files/attachedFiles.svelte' + +/** One tool as the settings modal lists it — the model-facing name, description and + * argument schema, which is exactly what the tool definition sends. */ +export type ToolSummary = { + name: string + description: string + /** The JSON Schema of the tool's arguments. `required` is defaulted because + * `SchemaViewer` reads it to mark the rows and renders nothing without it, and a + * tool whose arguments are all optional legitimately omits it. */ + parameters: Record +} + +/** Tool definitions as the modal lists them: name-sorted, so a list of dozens is + * scannable and stays put as the set changes between turns. */ +export function summarizeTools(tools: readonly Tool[]): ToolSummary[] { + return tools + .map((t) => ({ + name: t.def.function.name, + description: t.def.function.description ?? '', + parameters: { required: [], ...(t.def.function.parameters ?? {}) } + })) + .sort((a, b) => a.name.localeCompare(b.name)) +} + +/** Why an attachment is not reachable, or undefined when it is. The file tools operate + * on `readyFiles()`, so every other status is attached-but-unreadable and has to say so + * — a row that looks like the readable ones is the one place this could claim something + * the assistant cannot actually open. */ +export function attachmentStatusLabel(status: AttachedFileStatus): string | undefined { + switch (status) { + case 'ready': + return undefined + case 'locked': + return 'needs access' + case 'unavailable': + return 'unavailable' + case 'indexing': + return 'indexing…' + case 'error': + return 'failed' + } +} + +/** How many attachments the assistant can actually read, mirroring `readyFiles()`. + * + * A folder is counted on its children, never on its own status: that status is an + * aggregate, so one indexing child would hide the readable rest, while an empty or + * all-binary folder keeps a `ready` placeholder that `readyFiles()` filters out and + * reads as usable while exposing nothing. */ +export function countReadyAttachments( + folders: readonly { files: readonly { status: AttachedFileStatus }[] }[], + files: readonly { status: AttachedFileStatus }[] +): number { + const isReady = (f: { status: AttachedFileStatus }) => f.status === 'ready' + return folders.filter((d) => d.files.some(isReady)).length + files.filter(isReady).length +} diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 5ab271cc6f..634341b966 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -18,6 +18,7 @@ import { type AppCodeSelectionElement, type AppDatatableElement } from '../context' +import { appDatatableRole, sdkDatatableCall } from '$lib/components/raw_apps/dataTableRefUtils' // Backend runnable types export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline' @@ -921,9 +922,20 @@ export function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSy const policy = aiChatManager.datatableCreationPolicy const datatableName = policy.datatable ?? 'main' const schemaPrefix = policy.schema ? `${policy.schema}.` : '' - // Use wmill.datatable() for 'main' (default), otherwise wmill.datatable('name') - const datatableCall = - datatableName === 'main' ? 'wmill.datatable()' : `wmill.datatable('${datatableName}')` + // A role names the privileges the app's queries run with, so it has to be in the code the + // model writes. + const datatableRole = appDatatableRole(policy.roles, datatableName) + const tsDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'typescript') + const pyDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'python') + const roleEntries = Object.entries(policy.roles ?? {}) + const rolesNote = + roleEntries.length > 0 + ? `\n\nThis app uses these data tables through a role: ${roleEntries + .map(([dt, role]) => `\`${dt}\` as \`${role}\``) + .join( + ', ' + )}. Always pass that role when calling \`wmill.datatable\` on them, as in the examples. The role only reaches what it was granted, so a query on a table it lacks privileges on fails with \`permission denied\`.` + : '' let content = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend. @@ -1024,7 +1036,7 @@ Backend runnables should only perform **data operations** (SELECT, INSERT, UPDAT import * as wmill from 'windmill-client'; export async function main(user_id: string) { - const sql = ${datatableCall}; + const sql = ${tsDatatableCall}; const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne(); return user; } @@ -1035,12 +1047,12 @@ export async function main(user_id: string) { import wmill def main(user_id: str): - db = ${datatableCall} + db = ${pyDatatableCall} user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one() return user \`\`\` -Use these examples for normal datatable access. +Use these examples for normal datatable access.${rolesNote} ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte new file mode 100644 index 0000000000..7db5604775 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte @@ -0,0 +1,46 @@ + + + +{#if source} + + {#key content} + + {/key} +{:else} + +
    +
    + +
    +{/if} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte new file mode 100644 index 0000000000..9fc5f767b4 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte @@ -0,0 +1,42 @@ + + + + diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte new file mode 100644 index 0000000000..4924473e98 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte @@ -0,0 +1,196 @@ + + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
    +
    + Share with workspace + + {#if share} + Members of {workspace} can open a read-only copy of v{share.version} with this link. + {:else} + Members of {workspace} will be able to open a read-only copy of v{version} with a link. + {/if} + {#if status.current} + The copy is deleted {formatRetention(status.current.retention_secs)} after it is shared. + {/if} + +
    + + {#if share && url} +
    + + + Expires {displayDate(share.expires_at)} + +
    + {#if change} +
    + + {#if change === 'newer'} + The link shows v{share.version}; v{version} is on screen. + {:else if change === 'older'} + The link shows v{share.version}, newer than the v{version} on screen. + {:else} + The link still shows the old name, “{share.name}”. + {/if} + + +
    + {/if} +
    + +
    + {:else} +
    + +
    + {/if} +
    + {/snippet} +
    diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte index ebc46178d0..8331825ee2 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte @@ -1,23 +1,14 @@
    @@ -169,27 +144,22 @@ {/if}
    - - + /> + {#if canPreview} {#if restoringPin} - {:else if source} - - {#key `${artifact.id}:${pinnedContent ? `v${pinnedContent.version}` : artifact.updatedAt}`} - - {/key} {:else} - -
    -
    - -
    + {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts new file mode 100644 index 0000000000..115343117a --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/base', () => ({ base: '' })) + +import { shareWorkspaceId } from './artifactSharing' + +describe('shareWorkspaceId', () => { + it('shares a fork session into the topmost workspace the user still belongs to', () => { + const workspaces = [ + { id: 'prod' }, + { id: 'wm-fork-a', parent_workspace_id: 'prod' }, + { id: 'wm-fork-b', parent_workspace_id: 'wm-fork-a' } + ] + expect(shareWorkspaceId('wm-fork-b', workspaces)).toBe('prod') + }) + + it('stops below a parent the user is not a member of', () => { + const workspaces = [{ id: 'wm-fork-a', parent_workspace_id: 'prod' }] + expect(shareWorkspaceId('wm-fork-a', workspaces)).toBe('wm-fork-a') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts new file mode 100644 index 0000000000..57b08cba9b --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts @@ -0,0 +1,42 @@ +import { base } from '$lib/base' + +/** + * The workspace a share from `workspaceId` lands in: the topmost ancestor the user still + * belongs to. A session often runs in a fork, whose members are its creator alone, so a link + * minted there would reach nobody — and it would be deleted with the fork. + */ +export function shareWorkspaceId( + workspaceId: string, + workspaces: { id: string; parent_workspace_id?: string | null }[] +): string { + let current = workspaceId + const seen = new Set([current]) + for (;;) { + const parent = workspaces.find((w) => w.id === current)?.parent_workspace_id + if (!parent || seen.has(parent) || !workspaces.some((w) => w.id === parent)) return current + seen.add(parent) + current = parent + } +} + +export function sharedArtifactUrl(workspaceId: string, shareId: string): string { + return `${window.location.origin}${base}/shared_artifacts/${encodeURIComponent( + shareId + )}?workspace=${encodeURIComponent(workspaceId)}` +} + +/** "30 days", "12 hours": the retention window in the largest whole unit it fills. */ +export function formatRetention(secs: number): string { + const units: [string, number][] = [ + ['day', 86400], + ['hour', 3600], + ['minute', 60] + ] + for (const [unit, size] of units) { + if (secs >= size) { + const n = Math.floor(secs / size) + return `${n} ${unit}${n === 1 ? '' : 's'}` + } + } + return `${secs} second${secs === 1 ? '' : 's'}` +} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts index d70566ad18..a0fb5a4a4b 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts @@ -220,7 +220,7 @@ describe('artifactsDB', () => { expect(await noDb.getArtifact('a1')).toBeUndefined() expect(await noDb.listArtifactsForSession('s1')).toEqual([]) await expect(noDb.deleteArtifact('a1')).resolves.toBeUndefined() - await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBeUndefined() + await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBe(false) }) it('rejects a version read it could not make, instead of reading as absent', async () => { diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts index 94afac404e..f0cb32d71f 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts @@ -2,6 +2,8 @@ // active chat's rotation, so chatId-keying would drop artifacts on each new conversation. import { type DBSchema as IDBSchema, type IDBPObjectStore, type IDBPTransaction } from 'idb' import { userScopedDb } from '$lib/userScopedDb' +import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' export type ArtifactKind = 'md' | 'html' @@ -89,9 +91,11 @@ interface ArtifactsSchema extends IDBSchema { } } +const ARTIFACTS_DB = 'copilot-artifacts' + // User-scoped like the chat-history store these are keyed against: no cross-user // co-residency on a shared browser. -const dbh = userScopedDb('copilot-artifacts', { +const dbh = userScopedDb(ARTIFACTS_DB, { version: 2, // Runs for a fresh database and for the v1 upgrade alike, so create each store only // when it is missing. @@ -118,11 +122,69 @@ export async function putArtifact(artifact: PersistedArtifact): Promise { // A rejected write (most likely QuotaExceededError) leaves the artifact usable for the // session but unpersisted — degrade like the reads rather than throwing at the caller. await db.put('items', artifact) + markSessionDirty(artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not persist artifact', err) } } +/** A session's artifacts with their history, or undefined when the store is unavailable + * or no longer the named user's (see `readStoredSessions`). */ +export async function readSessionArtifacts( + sessionId: string, + email: string +): Promise<{ items: PersistedArtifact[]; versions: ArtifactVersion[] } | undefined> { + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return undefined + try { + const items = await db.getAllFromIndex('items', 'by-session', sessionId) + const versions = ( + await Promise.all(items.map((i) => db.getAllFromIndex('versions', 'by-artifact', i.id))) + ).flat() + return { items, versions } + } catch (err) { + console.error('Could not read artifacts', err) + return undefined + } +} + +/** Write restored artifacts and snapshots, leaving any that already exist alone. Reports + * whether they are all in the store now: unlike the other writes here, a caller records the + * restore as done on the strength of this answer. */ +export async function importArtifacts( + items: PersistedArtifact[], + versions: ArtifactVersion[], + email: string, + overwrite = false +): Promise { + if (items.length === 0 && versions.length === 0) return true + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const itemStore = tx.objectStore('items') + const versionStore = tx.objectStore('versions') + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const item of items) { + const existing = await itemStore.get(item.id) + if (existing === undefined || (overwrite && existing.updatedAt <= item.updatedAt)) { + await itemStore.put(item) + } + } + for (const version of versions) { + if (overwrite || (await versionStore.getKey(version.key)) === undefined) { + await versionStore.put(version) + } + } + await tx.done + return true + } catch (err) { + console.error('Could not import artifacts', err) + return false + } +} + export async function getArtifact(id: string): Promise { const db = await getDB() if (!db) return undefined @@ -273,7 +335,11 @@ export async function mutateArtifact( reportFailure = false abort() } - return { outcome: await settled, artifact: edit.artifact } + const outcome = await settled + if (outcome === 'saved' && db) { + markSessionDirty(edit.artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) + } + return { outcome, artifact: edit.artifact } } /** @@ -342,17 +408,55 @@ export async function deleteArtifact(id: string): Promise { if (!db) return try { const tx = db.transaction(['items', 'versions'], 'readwrite') - await tx.objectStore('items').delete(id) + const items = tx.objectStore('items') + const sessionId = (await items.get(id))?.sessionId + await items.delete(id) await deleteVersionsIn(tx.objectStore('versions'), id) await tx.done + if (sessionId) markSessionDirty(sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not delete artifact', err) } } -export async function deleteArtifactsForSession(sessionId: string): Promise { +/** Deletes these artifacts of the session (with their versions) and these versions: what + * an earlier restore staged for it and the backup no longer has. False when nothing could + * be deleted. */ +export async function pruneSessionArtifacts( + sessionId: string, + itemIds: Set, + versionKeys: Set, + email: string +): Promise { + if (itemIds.size === 0 && versionKeys.size === 0) return true const db = await getDB() - if (!db) return + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const items = tx.objectStore('items') + const versions = tx.objectStore('versions') + for (const id of await items.index('by-session').getAllKeys(sessionId)) { + if (!itemIds.has(String(id))) continue + await items.delete(id) + await deleteVersionsIn(versions, String(id)) + } + for (const key of versionKeys) await versions.delete(key) + await tx.done + return true + } catch (err) { + console.error('Could not prune artifacts for session', err) + return false + } +} + +/** False when the store could not be reached or the deletion failed. With `email`, only that + * user's store is touched: a caller that captured its user must not follow an account switch. */ +export async function deleteArtifactsForSession( + sessionId: string, + email?: string +): Promise { + const db = await getDB() + if (!db || (email !== undefined && db.name !== scopedKeyFor(ARTIFACTS_DB, email))) return false try { const tx = db.transaction(['items', 'versions'], 'readwrite') const items = tx.objectStore('items') @@ -365,8 +469,10 @@ export async function deleteArtifactsForSession(sessionId: string): Promise { + if (this.#sessionId === undefined) return + await this.#load() + } + async #load(): Promise { const token = ++this.#seq const id = this.#sessionId diff --git a/frontend/src/lib/components/copilot/chat/blobUtils.test.ts b/frontend/src/lib/components/copilot/chat/blobUtils.test.ts new file mode 100644 index 0000000000..fed92e9e4c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/blobUtils.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' +import { matchesAccept } from './blobUtils' + +function file(name: string, type: string): File { + return new File(['x'], name, { type }) +} + +describe('matchesAccept', () => { + it('matches an extension, a type wildcard and an exact media type', () => { + expect(matchesAccept(file('report.PDF', ''), '.pdf')).toBe(true) + expect(matchesAccept(file('shot.png', 'image/png'), 'image/*')).toBe(true) + expect(matchesAccept(file('shot.png', 'image/png'), 'image/png')).toBe(true) + }) + + it('refuses a file no pattern covers, and allows everything when the list is empty', () => { + expect(matchesAccept(file('notes.txt', 'text/plain'), '.pdf, image/*')).toBe(false) + expect(matchesAccept(file('notes.txt', 'text/plain'), '')).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/blobUtils.ts b/frontend/src/lib/components/copilot/chat/blobUtils.ts new file mode 100644 index 0000000000..41b5c76183 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/blobUtils.ts @@ -0,0 +1,58 @@ +/** + * Message attachments kept as their original bytes, such as a PDF, for a host that + * forwards them to object storage. Images are re-encoded and text files decoded instead, + * so neither lane can carry what the user picked unchanged. + */ + +/** Blobs one message may carry — the same slot cap images and text files use. */ +export const MAX_ATTACHED_BLOBS = 8 + +/** + * Per-blob byte cap. The data URL sits in composer state until send, so this + * bounds what one message can hold in memory; a host uploading elsewhere pays + * the same bytes again on the wire. + */ +export const MAX_BLOB_BYTES = 20_000_000 + +export type AttachedBlob = { + name: string + /** The file's own media type, verbatim — the upload's Content-Type depends on it. */ + mediaType: string + /** `data:;base64,<...>` of the original bytes. */ + dataUrl: string + size: number +} + +/** + * Whether a file satisfies an `accept` list — the same list the OS picker gets, applied + * again on drop, where the browser enforces nothing. + */ +export function matchesAccept(file: File, accept: string): boolean { + const patterns = accept + .split(',') + .map((p) => p.trim().toLowerCase()) + .filter(Boolean) + if (patterns.length === 0) return true + const type = file.type.toLowerCase() + const name = file.name.toLowerCase() + return patterns.some((pattern) => { + if (pattern.startsWith('.')) return name.endsWith(pattern) + if (pattern.endsWith('/*')) return type.startsWith(pattern.slice(0, -1)) + return type === pattern + }) +} + +export async function fileToAttachedBlob(file: File): Promise { + const dataUrl = await new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result)) + reader.onerror = () => reject(reader.error ?? new Error(`Could not read ${file.name}`)) + reader.readAsDataURL(file) + }) + return { + name: file.name, + mediaType: file.type || 'application/octet-stream', + dataUrl, + size: file.size + } +} diff --git a/frontend/src/lib/components/copilot/chat/chatViewHost.ts b/frontend/src/lib/components/copilot/chat/chatViewHost.ts new file mode 100644 index 0000000000..5c4dceb7f1 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/chatViewHost.ts @@ -0,0 +1,177 @@ +import { getContext, setContext } from 'svelte' +import type { AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { getAiChatManager } from './aiChatManagerContext' +import type { DisplayMessage, Tool } from './shared' +import type { ContextElement } from './context' +import type { AttachedBlob } from './blobUtils' +import type { AttachedImage } from './imageUtils' +import type { AttachedTextFile } from './textFileUtils' +import type { PasteAttachment } from './pasteTokens' +import type { AttachedFilesStore } from './files/attachedFiles.svelte' +import type { SessionArtifactsStore } from './artifacts/artifactsState.svelte' +import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' +import type { FlowAIChatHelpers } from './flow/core' +import type { AppAIChatHelpers } from './app/core' +import type AIChatInput from './AIChatInput.svelte' + +export type ChatSendRequestOptions = { + instructions?: string + pastes?: PasteAttachment[] + images?: AttachedImage[] + files?: AttachedTextFile[] + blobs?: AttachedBlob[] + /** Selected-context snapshot for this turn, in place of the live selection. Set + * whenever a send settles its context ahead of the turn. A host with no context + * of its own ignores it. */ + contextOverride?: ContextElement[] + /** Where `contextOverride` came from. 'pinned': chips picked for THIS message, so + * they are consumed from the live selection on send. 'replay': an edit or retry + * resending an older message's context, already consumed long ago. */ + contextOverrideOrigin?: 'pinned' | 'replay' +} + +/** + * What the chat view components (AIChatDisplay and everything it renders) need + * from whatever is driving the conversation. AIChatManager implements it for the + * copilot's own LLM loop; FlowChatViewHost implements it over a flow run's + * conversation so both chats render through the same components. + * + * Each affordance is gated by the field that answers for it — attachments by + * `supportsMessageAttachments`, the model button by `supportsModelSettings`, and so on — + * so a host turns on exactly what it can serve, and a new one is a matter of answering + * these fields rather than of being a copilot. `mode` is the exception, still read + * directly for chrome that only the copilot has. + */ +export interface ChatViewHost { + // Transcript + displayMessages: DisplayMessage[] + /** API-level messages. Only the count is read (context usage visibility). */ + messages: readonly unknown[] + contextTokens: number + /** The workspace a message's paths and jobs resolve against, which a fork session + * pins away from the navigated one. */ + readonly operatingWorkspace: string | undefined + loading: boolean + /** A send waiting for the chat's transcript to be restored; queue behind it. */ + sendPending: boolean + /** A turn this tab can neither follow nor stop, held by another tab on the same chat. */ + readonly runHeldElsewhere: boolean + loadingLabel: string | undefined + compacting: boolean + currentReply: string + currentReasoning: string + currentReasoningActive: boolean + readonly reasoningHiddenIndicatorLabel: string | undefined + readonly automaticScroll: boolean + enableAutomaticScroll: () => void + disableAutomaticScroll: () => void + + // Composer + instructions: string + readonly sendInFlight: boolean + /** Resolves to whether the draft was consumed as a turn. */ + sendRequest: (options?: ChatSendRequestOptions) => Promise + cancel: (reason?: string) => void + setAiChatInput: (aiChatInput: AIChatInput | null) => void + readonly queuedMessage: string + queuedContext: ContextElement[] | undefined + readonly queuedImages: AttachedImage[] + readonly queuedFiles: AttachedTextFile[] + readonly queuedBlobs: AttachedBlob[] + queueMessage: ( + text: string, + images?: AttachedImage[], + context?: ContextElement[], + files?: AttachedTextFile[], + blobs?: AttachedBlob[] + ) => void + dequeueMessage: () => void + setComposerStaged: (key: string, editingIndex: number | null, bytes: number) => void + /** Whether this composer holds anything unsent (text, pastes, attachments, reads in flight). */ + setComposerHasDraft: (key: string, hasDraft: boolean) => void + clearComposerStaged: (key: string) => void + attachmentBytesExcluding: (selfKey: string) => number + + // Per-message actions + storedImages: (displayMessageIndex: number) => AttachedImage[] | undefined + retryRequest: (messageIndex: number) => void + restartGeneration: ( + displayMessageIndex: number, + newContent?: string, + pastes?: PasteAttachment[], + images?: AttachedImage[], + editedContext?: ContextElement[], + files?: AttachedTextFile[] + ) => void | Promise + handleUserQuestionAnswer: (toolId: string, choices: string[]) => boolean + handleToolConfirmation: (toolId: string, confirmed: boolean) => void + /** A tool is waiting on a run form the user is filling in. Escape belongs to that form + * then, not to the turn — see AIChatDisplay's window handler. */ + readonly hasPendingRunForm: boolean + isRunFormPending: (toolCallId: string) => boolean + + // Copilot-only surfaces. Left undefined/false by hosts that have no LLM loop + // of their own; the chrome they drive hides itself. + mode?: AIMode + isSessionChat: boolean + /** Model + reasoning picker. Off where the model is configured elsewhere. */ + supportsModelSettings: boolean + /** Click a user message to edit and resend it. Needs a host that can rewind + * its own transcript, which a host replaying a server-side run cannot. */ + supportsMessageEditing: boolean + /** The `+` menu's file entry and drag-and-drop onto the panel. Attachments ride + * one message; where they go afterwards is the host's business (see sendRequest). */ + supportsMessageAttachments: boolean + /** + * Why attaching is off right now, when the host would otherwise take attachments. Distinct + * from `supportsMessageAttachments` being false, which means this chat never takes them: + * here the composer keeps the control and says what is missing, because moving the input + * elsewhere would only offer an editor that cannot work either. + */ + attachmentsUnavailableReason?: string + /** The turn needs text: attachments alone cannot be sent. True where the consumer + * requires a message of its own — an AI agent step refuses a run with neither a + * `user_message` nor manual memory. */ + requiresMessageText: boolean + /** The `+` menu's folder entries, backed by `attachedFiles`. A linked folder is a + * live handle on the user's disk, so only a host reading files in the browser has one. */ + supportsLinkedFolders: boolean + /** `accept` for the file picker, and the drop filter. A host whose consumer only + * understands some formats narrows it so the rest are refused rather than ignored. */ + attachmentAccept: string + /** How many attachments one turn can carry, when the consumer holds a fixed number — + * a flow input that is a single file, say. Undefined means no limit. Enforced at the + * picker and on drop, so what the composer shows is what the turn actually sends. */ + maxMessageAttachments?: number + /** Take non-image attachments verbatim (`blobs`) instead of decoding them to text. + * True where the bytes are forwarded somewhere — object storage — rather than read + * in the browser. */ + attachmentsAsBlobs: boolean + tools: Tool[] + autonomyMode: AIAutonomyMode + setAutonomyMode: (mode: AIAutonomyMode) => void + readonly autoAcceptEditsActive: boolean + readonly autoAcceptEditsAvailable: boolean + readonly autoAcceptToolConfirmationsAvailable: boolean + readonly planModeAvailable: boolean + attachedFiles: AttachedFilesStore + artifacts: SessionArtifactsStore + openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void + flowAiChatHelpers?: FlowAIChatHelpers + appAiChatHelpers?: AppAIChatHelpers +} + +const CHAT_VIEW_HOST_CONTEXT_KEY = 'chatViewHost' + +export function setChatViewHost(host: ChatViewHost) { + setContext(CHAT_VIEW_HOST_CONTEXT_KEY, host) +} + +/** + * Resolve the host driving the chat in this subtree. Falls back to the + * AIChatManager (scoped instance or app-wide singleton) so every existing + * copilot chat keeps working without setting anything. + */ +export function getChatViewHost(): ChatViewHost { + return getContext(CHAT_VIEW_HOST_CONTEXT_KEY) ?? getAiChatManager() +} diff --git a/frontend/src/lib/components/copilot/chat/composerBox.ts b/frontend/src/lib/components/copilot/chat/composerBox.ts new file mode 100644 index 0000000000..f85783d988 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/composerBox.ts @@ -0,0 +1,27 @@ +/** + * The composer's box, shared by both of AIChatInput's branches — the rich + * ContextTextarea and the plain textarea a host without @-context gets. + * + * Border and rounding live on the WRAPPER, never on the field, so the chip rows + * (context, files, images) sit inside the box above the text. The field's own + * @tailwindcss/forms border, ring and background are neutralised so only the + * wrapper reads as the input. + * + * The disabled treatment is on the wrapper for the same reason: `disabled` on the + * field alone leaves it looking exactly like a usable one, so the only cue that + * typing is refused is placeholder text the eye reads as an invitation. + */ + +const BOX_BASE = 'w-full scroll-pb-2 rounded-md border border-border-light transition-colors' + +export function composerBoxClass(disabled: boolean = false): string { + return `${BOX_BASE} ${ + disabled + ? 'bg-surface-disabled cursor-not-allowed' + : 'bg-surface-input focus-within:border-border-selected' + }` +} + +/** Applied to the field inside the box; without it the field draws a second border. */ +export const COMPOSER_FIELD_RESET = + '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0 disabled:cursor-not-allowed disabled:placeholder:text-disabled' diff --git a/frontend/src/lib/components/copilot/chat/context.ts b/frontend/src/lib/components/copilot/chat/context.ts index bf915515db..f57862c179 100644 --- a/frontend/src/lib/components/copilot/chat/context.ts +++ b/frontend/src/lib/components/copilot/chat/context.ts @@ -331,6 +331,13 @@ export function contextElementKey(c: ContextElement): string { return c.type === 'app_dom_selector' ? `dom:${c.appPath}:${c.selector}` : `${c.type}:${c.title}` } +/** An `@`-mentioned workspace item, attached to one message the way a DOM pick + * is. Membership only: this does not gate on mode, so every caller must. Outside + * GLOBAL these stay selected as chips the user removes by hand. */ +export function isMentionContext(c: ContextElement): boolean { + return c.type === 'workspace_script' || c.type === 'workspace_flow' || c.type === 'workspace_app' +} + export function isSameContextElement(a: ContextElement, b: ContextElement): boolean { if (a.type !== b.type) return false if (a.type === 'app_dom_selector' && b.type === 'app_dom_selector') { diff --git a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts index 725385d3ee..c76e892cdf 100644 --- a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts @@ -5,6 +5,10 @@ type MaybePromise = T | Promise type ToolDisplayActionHandler = (action: ToolDisplayAction) => MaybePromise const toolDisplayActionHandlers = $state>({}) +// Every registration per type, latest last: a page that takes over a type from the layout +// (the sessions page opens items in its panel rather than in drawers) hands it back on +// unmount instead of leaving the type unhandled. +const registrations: Record = {} function formatUnknownError(error: unknown): string { if (error instanceof Error) { @@ -17,14 +21,28 @@ export function registerToolDisplayActionHandler( type: ToolDisplayAction['type'], handler: ToolDisplayActionHandler ): () => void { + const stack = (registrations[type] ??= []) + stack.push(handler) toolDisplayActionHandlers[type] = handler return () => { - if (toolDisplayActionHandlers[type] === handler) { - delete toolDisplayActionHandlers[type] - } + const at = stack.lastIndexOf(handler) + if (at < 0) return + stack.splice(at, 1) + const current = stack[stack.length - 1] + if (current) toolDisplayActionHandlers[type] = current + else delete toolDisplayActionHandlers[type] } } +/** + * Reactive: reads the `$state` registry, so a component re-renders when a page mounts or + * unmounts its handler. Offering an action without checking this yields an affordance whose + * only outcome is the unavailable-action toast. + */ +export function hasToolDisplayActionHandler(type: ToolDisplayAction['type']): boolean { + return toolDisplayActionHandlers[type] !== undefined +} + export async function runToolDisplayAction(action: ToolDisplayAction): Promise { const handler = toolDisplayActionHandlers[action.type] if (!handler) { diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts index 7232500898..bb7976d396 100644 --- a/frontend/src/lib/components/copilot/chat/datatableTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { WorkspaceService, type CompletedJob } from '$lib/gen' import type { DataTableTables } from '$lib/gen/types.gen' import { runScript } from '$lib/components/jobs/utils' +import { datatableReference } from '$lib/components/dbTypes' import { createToolDef, executeTestRun, @@ -15,9 +16,9 @@ import { * * Datatables are workspace-level managed PostgreSQL databases. The backend * endpoints used here (`list_datatable_tables`, `get_datatable_table_schema`) - * and SQL execution (`datatable://`) are gated only by workspace - * membership, so these tools need no app context and operate directly on the - * workspace. This is the unrestricted counterpart to the app-mode datatable + * and SQL execution (`datatable://`) need no app context: the server + * decides what the caller reaches, as the datatable role they name or its + * default. This is the unrestricted counterpart to the app-mode datatable * tools in `app/core.ts`, which additionally filter by the app's whitelist. */ @@ -31,9 +32,19 @@ const memo = (factory: () => T): (() => T) => { // ============= Pure workspace-scoped operations ============= -/** List all datatables configured in the workspace, with their schema/table names. */ -export async function listDatatables(workspace: string): Promise { - return await WorkspaceService.listDataTableTables({ workspace }) +/** List the datatables configured in the workspace, with their schema/table names: all of them as + * their default role, or only `datatableName`, as `role` when one is given. */ +export async function listDatatables( + workspace: string, + datatableName?: string, + role?: string +): Promise { + if (datatableName === undefined) return await WorkspaceService.listDataTableTables({ workspace }) + return await WorkspaceService.listDataTableTables({ + workspace, + datatableName, + ...(role !== undefined && { roleFor: datatableName, role }) + }) } /** Get the columns (column_name -> compact_type) of one datatable table. */ @@ -41,13 +52,15 @@ export async function getDatatableColumns( workspace: string, datatableName: string, schemaName: string, - tableName: string + tableName: string, + role?: string ): Promise> { const schema = await WorkspaceService.getDataTableTableSchema({ workspace, datatableName, schemaName, - tableName + tableName, + role }) return schema.columns } @@ -81,7 +94,26 @@ const NO_DATATABLES_CONFIGURED_MESSAGE = // ============= Tool definitions ============= -const getListDatatablesSchema = memo(() => z.object({})) +// The same rule the server applies to `-- role `; a name it would refuse fails here instead. +const getRoleSchema = memo(() => + z + .string() + .regex(/^[A-Za-z0-9_-]{1,63}$/) + .optional() + .describe( + "The datatable role to connect as, when the code you are working on uses one (an app's `data.roles` entry, or the `role` it passes to wmill.datatable). Omit for the datatable's default role." + ) +) + +const getListDatatablesSchema = memo(() => + z.object({ + datatable_name: z + .string() + .optional() + .describe('List only this datatable. Required with `role`.'), + role: getRoleSchema() + }) +) const getListDatatablesToolDef = memo(() => createToolDef( getListDatatablesSchema(), @@ -94,7 +126,8 @@ const getGetDatatableTableSchemaSchema = memo(() => z.object({ datatable_name: z.string().describe('The datatable name to inspect, e.g. "main".'), schema_name: z.string().describe('The schema name, e.g. "public".'), - table_name: z.string().describe('The table name to inspect.') + table_name: z.string().describe('The table name to inspect.'), + role: getRoleSchema() }) ) const getGetDatatableTableSchemaToolDef = memo(() => @@ -117,6 +150,7 @@ const getExecDatatableSqlSchema = memo(() => .describe( 'The SQL query to execute. Supports SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, ALTER TABLE, DROP TABLE, etc. For SELECT queries, results are returned as an array of objects. A newly created table will appear in list_datatables automatically.' ), + role: getRoleSchema(), background: z .boolean() .optional() @@ -217,10 +251,18 @@ export function getDatatableTools(): Tool<{}>[] { { def: getListDatatablesToolDef(), planModeSafe: true, - fn: async ({ workspace, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing datatables...' }) try { - const metadata = await listDatatables(workspace) + const parsedArgs = getListDatatablesSchema().parse(args ?? {}) + if (parsedArgs.role !== undefined && parsedArgs.datatable_name === undefined) { + throw new Error('`role` needs `datatable_name`, the datatable it is a role of') + } + const metadata = await listDatatables( + workspace, + parsedArgs.datatable_name, + parsedArgs.role + ) if (metadata.length === 0) { toolCallbacks.setToolStatus(toolId, { content: 'No datatables configured — set one up in workspace settings' @@ -236,7 +278,18 @@ export function getDatatableTools(): Tool<{}>[] { toolCallbacks.setToolStatus(toolId, { content: `Listed ${metadata.length} datatable(s) with ${totalTables} table(s)` }) - return JSON.stringify(metadata, null, 2) + // Only what the model acts on: the roles it may pass, not the creation privileges + // the manager's UI gates on. + return JSON.stringify( + metadata.map((d) => ({ + datatable_name: d.datatable_name, + schemas: d.schemas, + ...(d.error && { error: d.error }), + ...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role }) + })), + null, + 2 + ) } catch (e) { const errorMsg = `Error listing datatables: ${e instanceof Error ? e.message : String(e)}` toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) @@ -257,7 +310,8 @@ export function getDatatableTools(): Tool<{}>[] { workspace, parsedArgs.datatable_name, parsedArgs.schema_name, - parsedArgs.table_name + parsedArgs.table_name, + parsedArgs.role ) toolCallbacks.setToolStatus(toolId, { content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` @@ -300,7 +354,7 @@ export function getDatatableTools(): Tool<{}>[] { requestBody: { language: 'postgresql', content: parsedArgs.sql, - args: { database: `datatable://${name}` } + args: { database: datatableReference(name, parsedArgs.role) } } }), workspace, diff --git a/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts b/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts index fdd8c74558..d02d1b9497 100644 --- a/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/ducklakeTools.test.ts @@ -1,6 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { listMock } = vi.hoisted(() => ({ listMock: vi.fn() })) +const { listMock, listMetricsMock } = vi.hoisted(() => ({ + listMock: vi.fn(), + listMetricsMock: vi.fn() +})) vi.mock('./shared', () => ({ createToolDef: (_schema: unknown, name: string, description: string) => ({ @@ -10,7 +13,8 @@ vi.mock('./shared', () => ({ })) vi.mock('$lib/gen', () => ({ - WorkspaceService: { listDucklakes: listMock } + WorkspaceService: { listDucklakes: listMock }, + DataMetricService: { listDataMetrics: listMetricsMock } })) import { getDucklakeTools } from './ducklakeTools' @@ -31,7 +35,10 @@ function run(name: string, args: Record = {}) { }) } -beforeEach(() => listMock.mockReset()) +beforeEach(() => { + listMock.mockReset() + listMetricsMock.mockReset() +}) describe('list_ducklakes', () => { it('returns the configured catalog names', async () => { @@ -51,3 +58,65 @@ describe('list_ducklakes', () => { expect(result).toContain('still draft the pipeline scripts') }) }) + +describe('list_data_metrics', () => { + it('forwards the filters and returns the declarations', async () => { + listMetricsMock.mockResolvedValue({ + metrics: [ + { + script_path: 'f/analytics/rev', + table_path: 'main/main.orders', + kind: 'measure', + name: 'revenue', + expr: 'sum(amount)', + filter: 'not is_test' + } + ] + }) + const result = await run('list_data_metrics', { + table: 'ducklake://main/main.orders', + path_prefix: 'f/analytics', + limit: 50 + }) + expect(listMetricsMock).toHaveBeenCalledWith({ + workspace: 'test-workspace', + table: 'ducklake://main/main.orders', + pathPrefix: 'f/analytics', + perPage: 50 + }) + expect(JSON.parse(result).metrics[0]).toMatchObject({ name: 'revenue', expr: 'sum(amount)' }) + }) + + it('never reports an empty result as proof that nothing is declared', async () => { + listMetricsMock.mockResolvedValue({ metrics: [] }) + const result = await run('list_data_metrics', {}) + // Unreadable declarations are omitted, not flagged, so absence is unprovable. + expect(result).toContain('does not establish') + expect(result).toContain('cannot read') + }) + + // The server matches a lake-less name against nothing, so the readability hedge + // would confirm "nothing is declared" for a filter worth retrying. The scheme is + // optional on the way in, so the retry it names must not carry it back. + it.each(['orders', 'ducklake://orders'])( + 'blames the missing lake, not readability, for table %s', + async (table) => { + listMetricsMock.mockResolvedValue({ metrics: [] }) + const result = await run('list_data_metrics', { table }) + expect(result).toContain('`/orders`') + expect(result).not.toContain('cannot read') + } + ) + + it('warns that more declarations exist when the page is cut short', async () => { + listMetricsMock.mockResolvedValue({ + // A cursor only comes back on a full page, never with an empty one. + metrics: [{ table_path: 't', kind: 'measure', name: 'n', script_path: 's' }], + next_cursor: { table_path: 't', kind: 'measure', name: 'n', script_path: 's' } + }) + const result = await run('list_data_metrics', {}) + // Without this the model reads a partial page as "no such measure" and + // re-derives a number that disagrees with the declared one. + expect(result).toContain('rather than concluding a measure is undeclared') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/ducklakeTools.ts b/frontend/src/lib/components/copilot/chat/ducklakeTools.ts index 5594e4f3d0..dfadc58db9 100644 --- a/frontend/src/lib/components/copilot/chat/ducklakeTools.ts +++ b/frontend/src/lib/components/copilot/chat/ducklakeTools.ts @@ -1,17 +1,19 @@ import { z } from 'zod' -import { WorkspaceService } from '$lib/gen' +import { DataMetricService, WorkspaceService } from '$lib/gen' import { createToolDef, type Tool } from './shared' /** - * Workspace-scoped DuckLake readiness tool, the pipeline counterpart to - * `list_datatables` in `datatableTools.ts`. + * Workspace-scoped DuckLake tools, the pipeline counterpart to `list_datatables` + * in `datatableTools.ts`. * * A data pipeline materializes DuckLake tables and reads/writes S3 assets, which * only work once the workspace has object storage + a DuckLake catalog - * configured. This tool lets the chat detect that prerequisite (and warn with - * role-appropriate next steps) instead of silently producing a pipeline that - * cannot run. It is a plain read gated only by workspace membership, so it needs - * no app context and belongs in the global tool set. + * configured. `list_ducklakes` lets the chat detect that prerequisite (and warn + * with role-appropriate next steps) instead of silently producing a pipeline that + * cannot run. `list_data_metrics` reads the declarations recorded against those + * lake tables, not the tables themselves. Both are plain reads gated only by + * workspace membership, so they need no app context and belong in the global tool + * set. */ /** List the names of the DuckLake catalogs configured in the workspace. */ @@ -31,9 +33,78 @@ const listDucklakesToolDef = createToolDef( 'List the DuckLake catalogs configured in this workspace, by name. Call this before building or deploying a data pipeline that materializes DuckLake tables or reads/writes S3 assets: if it returns none, the workspace has no object storage + DuckLake configured and the pipeline cannot run until a workspace admin sets it up. Returns names only.' ) +const listDataMetricsSchema = z.object({ + table: z + .string() + .optional() + .describe( + 'Only declarations on this DuckLake table, as `/
    Triggers to deploy
    +
    +
    + + {#if trigger.isPrimary} + + {/if} +
    + +
    + +
    +
    +
    + {#if permission === 'deploy'} +
    + toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
    + {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
    Agents to deploy
    +
    + +
    +
    + + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
    + {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
    +
    +
    + {#if permission.state === 'deploy'} +
    + toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
    + {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
    Triggers to deploy
    -
    -
    - - {#if trigger.isPrimary} - - {/if} -
    -
    - -
    -
    -
    - {#if permission === 'deploy'} -
    - toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
    - {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
    - No draft triggers found -
    ` or `/.
    `, with or without the `ducklake://` scheme. A name with no lake matches nothing and comes back empty.' + ), + path_prefix: z + .string() + .optional() + .describe('Only declarations made by scripts under this path, e.g. `f/analytics`.'), + limit: z + .number() + .int() + .min(1) + .max(1000) + .optional() + .describe('Max number of declarations to return. Defaults to 200.') +}) +const listDataMetricsToolDef = createToolDef( + listDataMetricsSchema, + 'list_data_metrics', + 'List the measures and dimensions declared on DuckLake tables (from `// measure` / `// dimension` annotations in deployed scripts). Call this before writing any aggregate query over a DuckLake table: a declared measure is the canonical definition of that number, and reproducing it yourself silently disagrees with it (a `revenue` measure typically excludes refunds or test rows). Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates share one GROUP BY. Only declarations whose producing script you can read are returned, so what comes back is never proof of what exists: if the number you need is not here you may still write your own aggregate, but say that you found no declared measure for it rather than implying none exists.' +) + +// The endpoint drops declarations whose producing script the caller cannot read +// (token scope + RLS on `script`), so an empty result means "none declared" or +// "none readable by you" and the tool cannot tell which. +const NO_DATA_METRICS_NOTE = + 'Nothing matched. That does not establish that nothing is declared: declarations whose producing script you cannot read are omitted from this list, not flagged. You may write your own aggregate, but tell the user you found no declared measure you can read rather than stating none is declared.' + +// Well under the endpoint's 1000 cap: a full page is pretty-printed into the +// chat context, and 1000 declarations would cost tens of thousands of tokens. +const DEFAULT_DATA_METRICS_LIMIT = 200 + /** The workspace DuckLake tools, for registration in global mode. */ export function getDucklakeTools(): Tool<{}>[] { return [ + { + def: listDataMetricsToolDef, + planModeSafe: true, + showDetails: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const parsed = listDataMetricsSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: 'Listing declared measures...' }) + const limit = parsed.limit ?? DEFAULT_DATA_METRICS_LIMIT + const { metrics, next_cursor } = await DataMetricService.listDataMetrics({ + workspace, + table: parsed.table, + pathPrefix: parsed.path_prefix, + perPage: limit + }) + // `canonical_table_path` expands a value only once it contains a `/`, so a bare + // table name matches nothing — a retryable filter, not a permissions outcome. + const bareTable = parsed.table?.replace('ducklake://', '') + const emptyNote = + bareTable && !bareTable.includes('/') + ? `Nothing matched \`${parsed.table}\`: a table filter must name its lake, as \`/${bareTable}\`. Re-call with the lake before concluding anything about what is declared.` + : NO_DATA_METRICS_NOTE + const note = next_cursor + ? `More declarations exist beyond the first ${limit}. Re-call with table/path_prefix to target what you are looking for, or with a higher limit (max 1000) for the rest of the list, rather than concluding a measure is undeclared.` + : metrics.length === 0 + ? emptyNote + : undefined + const result = JSON.stringify({ metrics, ...(note ? { note } : {}) }, null, 2) + toolCallbacks.setToolStatus(toolId, { + content: `Listed ${metrics.length} declared measure(s)/dimension(s)`, + result + }) + return result + } + }, { def: listDucklakesToolDef, planModeSafe: true, diff --git a/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts new file mode 100644 index 0000000000..e257e40975 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts @@ -0,0 +1,106 @@ +import { get } from 'svelte/store' +import { userStore } from '$lib/stores' + +/** + * Which workspace-object paths the chat may act through, remembered per workspace + * and per account. + * + * Only the paths someone actually decided about are stored; everything else is the + * kind's default. That is what lets a default flip — skills went from off-until-on + * to on-until-off — without rewriting anyone's storage: the stored entries keep + * meaning what the person chose, and only the paths they never touched move. + * + * Stored per browser, like the chat's other per-user preferences, but keyed by + * email as well as workspace: browser storage outlives a logout, and one person's + * decisions must not be read as the next person's — whichever way they went. + * Workspace ids cannot contain `:`, so the composite key is unambiguous. + */ +export type PathsPreference = { + /** Paths stored as on, which is the enabled set only for a kind that defaults to + * off. For one that defaults to on this is whatever happens to be written down — + * nothing for a decision made since the default flipped, and the whole selection + * for a browser still carrying storage from before it. Neither is the answer to + * "what is enabled": ask `isEnabled` per path for that. */ + explicitlyEnabledPaths: (workspace: string) => string[] + isEnabled: (workspace: string, path: string) => boolean + /** Returns false when there is no account to record the preference against, so + * a caller that just created the object can say it did not stay on. */ + setEnabled: (workspace: string, path: string, enabled: boolean) => boolean + /** Drop any decision about `path`, leaving it at the default. What a caller wants + * when the thing at that path is gone — a deleted or renamed skill — rather than + * writing the default as a decision, which reads as one and would mean the + * opposite the day the default moves. */ + forget: (workspace: string, path: string) => void +} + +/** One decision per path. The older shape was an array of the paths that were on, + * which says exactly that and so needs no conversion to be read. */ +type Choices = Record + +function toChoices(raw: unknown): Choices { + if (Array.isArray(raw)) return Object.fromEntries(raw.map((path) => [String(path), true])) + if (raw && typeof raw === 'object') { + return Object.fromEntries( + Object.entries(raw as Record).map(([path, on]) => [path, on === true]) + ) + } + return {} +} + +export function createPathsPreference( + storageKey: string, + defaultEnabled: boolean +): PathsPreference { + function scope(workspace: string): string | undefined { + const email = get(userStore)?.email + return email ? `${workspace}:${email}` : undefined + } + + function read(): Record { + if (typeof localStorage === 'undefined') return {} + try { + return JSON.parse(localStorage.getItem(storageKey) ?? '{}') + } catch { + return {} + } + } + + function choices(workspace: string): Choices { + const key = scope(workspace) + return key ? toChoices(read()[key]) : {} + } + + /** `decision` of undefined drops the entry. */ + function write(workspace: string, path: string, decision: boolean | undefined): boolean { + const key = scope(workspace) + if (!key) return false + const all = read() + const current = toChoices(all[key]) + if (decision === undefined) { + delete current[path] + } else { + current[path] = decision + } + all[key] = current + try { + localStorage.setItem(storageKey, JSON.stringify(all)) + } catch (e) { + console.error(`Failed to persist ${storageKey}`, e) + } + return true + } + + return { + explicitlyEnabledPaths: (workspace) => + Object.entries(choices(workspace)) + .filter(([, on]) => on) + .map(([path]) => path), + isEnabled: (workspace, path) => choices(workspace)[path] ?? defaultEnabled, + // A path put back to the default is dropped rather than stored as one: the + // entries are the decisions, so a re-enabled skill leaves nothing behind and a + // later skill at that path starts from the default like any other. + setEnabled: (workspace, path, enabled) => + write(workspace, path, enabled === defaultEnabled ? undefined : enabled), + forget: (workspace, path) => void write(workspace, path, undefined) + } +} diff --git a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts index 9673e29edc..59d5d57204 100644 --- a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts +++ b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.test.ts @@ -19,7 +19,7 @@ describe('attachedFilesDB without IndexedDB', () => { putItem({ id: 'a', sessionId: 's1', kind: 'snapshot', name: 'x.txt', addedAt: 0 }) ).resolves.toBeUndefined() await expect(deleteItem('a')).resolves.toBeUndefined() - await expect(deleteItemsForSession('s1')).resolves.toBeUndefined() + await deleteItemsForSession('s1') }) it('does not throw when requesting persistent storage', async () => { diff --git a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts index fe051ef8cc..dc872845cd 100644 --- a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts +++ b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts @@ -91,9 +91,10 @@ export async function deleteItem(id: string): Promise { await db?.delete('items', id) } -export async function deleteItemsForSession(sessionId: string): Promise { +/** False when the store could not be reached or the deletion failed. */ +export async function deleteItemsForSession(sessionId: string): Promise { const db = await getDB() - if (!db) return + if (!db) return false try { const tx = db.transaction('items', 'readwrite') const index = tx.store.index('by-session') @@ -103,8 +104,10 @@ export async function deleteItemsForSession(sessionId: string): Promise { cursor = await cursor.continue() } await tx.done + return true } catch (err) { console.error('Could not delete attached files for session', err) + return false } } diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 47a6b0b169..9744085567 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -4,6 +4,7 @@ import type { ExtendedOpenFlow, FlowEditorContext } from '$lib/components/flows/types' import type { InputTransform } from '$lib/gen' import type { FlowAIChatHelpers } from './core' + import { chatMemoryId } from '../global/core' import { createInlineScriptSession } from './inlineScriptsUtils' import { loadSchemaFromModule } from '$lib/components/flows/flowInfers' import { getAiChatManager } from '../aiChatManagerContext' @@ -14,6 +15,9 @@ import type { ScriptLintResult } from '../shared' import { applyFlowJsonUpdate, updateRawScriptModuleContent } from './helperUtils' import { findModuleInFlow } from '$lib/components/flows/flowTree' + import { getEditorStoragePath } from '$lib/components/editorStoragePathContext' + + const editorStoragePath = getEditorStoragePath() let { flowModuleSchemaMap, @@ -162,13 +166,15 @@ selectionManager.selectId(id, { openPanel: true }) }, + getStoragePath: () => editorStoragePath?.(), + testFlow: async (args, conversationId) => { // Set preview args if provided if (args) { previewArgs.val = args } // Call the UI test function which opens preview panel - return await onTestFlow?.(conversationId) + return await onTestFlow?.(conversationId ?? chatMemoryId(flowStore.val.value)) }, getLintErrors: async (moduleId: string): Promise => { diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 63c4ffe94f..51fea34d58 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -136,6 +136,10 @@ export interface FlowAIChatHelpers { /** Run a test of the current flow using the UI's preview mechanism */ testFlow: (args?: Record, conversationId?: string) => Promise + /** The path this editor's draft is stored under. Tells a caller which of several mounted + * editors is the one an active-editor context names. */ + getStoragePath: () => string | undefined + /** Get lint errors from a specific module (focuses it first, waits for Monaco to analyze) */ getLintErrors: (moduleId: string) => Promise } diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index cab2839d48..b8f44d2635 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.791.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.813.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts index 5bd36b85ce..16a249e5ac 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts @@ -1,6 +1,6 @@ import { z } from "zod" -export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n").optional(), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("window"), "context_length": z.number().int().describe("Number of most recent messages to load and store. 0 turns memory off.") }).describe("Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.")]).describe("Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`."), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "memory_id": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n").optional(), "previous_messages": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "enabled_tools": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -17,10 +17,10 @@ export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "i message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type") -export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleSchema = z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.lazy(() => flowModuleSchema)).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n").optional(), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("window"), "context_length": z.number().int().describe("Number of most recent messages to load and store. 0 turns memory off.") }).describe("Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.")]).describe("Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`."), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "memory_id": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n").optional(), "previous_messages": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "enabled_tools": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -37,7 +37,7 @@ export const flowModuleSchema = z.object({ "id": z.string().describe("Unique ide message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") export const flowModulesSchema = z.array(flowModuleSchema) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts deleted file mode 100644 index 8158fe62b8..0000000000 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { listMcpToolsMock } = vi.hoisted(() => ({ - listMcpToolsMock: vi.fn() -})) - -vi.mock('../shared', () => ({ - createToolDef: (_schema: unknown, name: string, description: string) => ({ - type: 'function', - function: { name, description, parameters: {} } - }) -})) - -vi.mock('$lib/gen', () => ({ - McpService: { - listMcpTools: listMcpToolsMock - } -})) - -import { apiCatalogTools, clearApiCatalogCache } from './apiCatalogTools' - -const CATALOG = [ - { - name: 'listWorkers', - description: 'List workers', - instructions: 'List all workers with their ping status', - path: '/workers/list', - method: 'GET', - query_params_schema: { - type: 'object', - properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } - } - }, - { - name: 'getJob', - description: 'Get job details', - instructions: '', - path: '/w/{workspace}/jobs_u/get/{id}', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string' } }, - required: ['workspace', 'id'] - } - }, - { - name: 'deleteSchedule', - description: 'Delete a schedule', - instructions: '', - path: '/w/{workspace}/schedules/delete/{path}', - method: 'DELETE', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - } - }, - { - name: 'createFlow', - description: 'Create a flow', - instructions: '', - path: '/w/{workspace}/flows/create', - method: 'POST' - }, - { - name: 'getVariable', - description: 'Get variable', - instructions: '', - path: '/w/{workspace}/variables/get/{path}', - method: 'GET' - }, - { - name: 'getScriptByPath', - description: 'Get script by path', - instructions: '', - path: '/w/{workspace}/scripts/get/p/{path}', - method: 'GET' - }, - { - name: 'deleteScriptByHash', - description: 'Delete a script by hash', - instructions: '', - path: '/w/{workspace}/scripts/delete/h/{hash}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, hash: { type: 'string' } }, - required: ['workspace', 'hash'] - } - }, - { - name: 'runFlowByPath', - description: 'Run flow by path', - instructions: 'Trigger a run of a deployed flow', - path: '/w/{workspace}/jobs/run/f/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { - type: 'object', - properties: { args: { type: 'object' } } - } - } -] - -function createToolCallbacks() { - return { - setToolStatus: vi.fn(), - removeToolStatus: vi.fn() - } as any -} - -function getTool(name: string) { - const tool = apiCatalogTools.find((entry) => entry.def.function.name === name) - if (!tool) throw new Error(`${name} tool not found`) - return tool -} - -async function run(name: string, args: Record) { - const raw = await getTool(name).fn({ - args, - workspace: 'test-ws', - helpers: {}, - toolCallbacks: createToolCallbacks(), - toolId: 'tool-1' - }) - return JSON.parse(raw) -} - -beforeEach(() => { - vi.clearAllMocks() - clearApiCatalogCache() - listMcpToolsMock.mockResolvedValue(CATALOG) - vi.unstubAllGlobals() -}) - -describe('search_api_endpoints', () => { - it('matches on name/path tokens, plural-insensitively, and excludes covered endpoints', async () => { - const result = await run('search_api_endpoints', { query: 'worker' }) - expect(result.matches.map((m: any) => m.name)).toEqual(['listWorkers']) - expect(result.matches[0].endpoint).toBe('GET /workers/list') - expect(result.matches[0].params).toEqual(['page', 'per_page']) - expect(result.matches[0].instructions).toContain('ping status') - - const flows = await run('search_api_endpoints', { query: 'create flow' }) - expect(flows.matches.map((m: any) => m.name)).not.toContain('createFlow') - expect(flows.covered_by_dedicated_tools).toContain('createFlow → use write_flow') - }) - - it('returns endpoint categories when nothing matches', async () => { - const result = await run('search_api_endpoints', { query: 'kubernetes' }) - expect(result.matches).toEqual([]) - expect(result.hint).toContain('workers') - expect(result.hint).toContain('jobs') - }) -}) - -describe('call_api_get', () => { - it('rejects covered, unknown, and non-GET endpoints with a pointer', async () => { - const covered = await run('call_api_get', { name: 'createFlow' }) - expect(covered.error).toContain('write_flow') - - const unknown = await run('call_api_get', { name: 'nope' }) - expect(unknown.error).toContain('search_api_endpoints') - - const mutating = await run('call_api_get', { name: 'runFlowByPath' }) - expect(mutating.error).toContain('call_api_endpoint') - - const deleting = await run('call_api_endpoint', { name: 'deleteSchedule' }) - expect(deleting.error).toContain('delete_workspace_item') - - const byHash = await run('call_api_endpoint', { name: 'deleteScriptByHash' }) - expect(byHash.error).toContain('delete_workspace_item') - const search = await run('search_api_endpoints', { query: 'delete script' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('deleteScriptByHash') - }) - - it('refuses draft-blind item reads and lists, pointing at the draft-aware tools', async () => { - for (const name of ['getScriptByPath', 'getResource', 'getSchedule']) { - const result = await run('call_api_get', { name }) - expect(result.success).toBe(false) - expect(result.error).toContain('read_workspace_item') - } - for (const name of ['listScripts', 'listFlows', 'listResource', 'listSchedules']) { - const result = await run('call_api_get', { name }) - expect(result.success).toBe(false) - expect(result.error).toContain('list_workspace_items') - } - - const search = await run('search_api_endpoints', { query: 'get script' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('getScriptByPath') - }) - - it('refuses variable reads so variable values never reach the model', async () => { - const result = await run('call_api_get', { name: 'getVariable' }) - expect(result.success).toBe(false) - expect(result.error).toContain('never readable') - - const search = await run('search_api_endpoints', { query: 'variable' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('getVariable') - }) - - it('returns the endpoint schema when a required path param is missing', async () => { - const result = await run('call_api_get', { name: 'getJob' }) - expect(result.success).toBe(false) - expect(result.error).toContain('id') - expect(result.schema.path_params_schema.required).toContain('id') - }) - - it('substitutes path params and sends the rest as query params', async () => { - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - headers: new Headers({ 'content-type': 'application/json' }), - json: async () => [{ worker: 'w1' }] - }) - vi.stubGlobal('fetch', fetchMock) - const result = await run('call_api_get', { name: 'listWorkers', params: { page: 2 } }) - expect(fetchMock).toHaveBeenCalledWith('/api/workers/list?page=2', { method: 'GET' }) - expect(result).toEqual({ success: true, data: [{ worker: 'w1' }] }) - }) -}) - -describe('call_api_endpoint', () => { - it('requires confirmation and executes mutating endpoints with a body', async () => { - expect(getTool('call_api_endpoint').requiresConfirmation).toBe(true) - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - headers: new Headers({ 'content-type': 'text/plain' }), - text: async () => 'job-id-1' - }) - vi.stubGlobal('fetch', fetchMock) - const result = await run('call_api_endpoint', { - name: 'runFlowByPath', - params: { path: 'u/me/myflow' }, - body: { args: { n: 1 } } - }) - expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs/run/f/u%2Fme%2Fmyflow', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ args: { n: 1 } }) - }) - expect(result).toEqual({ success: true, data: 'job-id-1' }) - }) - - it('redirects GET endpoints to call_api_get', async () => { - const result = await run('call_api_endpoint', { name: 'listWorkers' }) - expect(result.error).toContain('call_api_get') - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts deleted file mode 100644 index 5d1324ebe3..0000000000 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts +++ /dev/null @@ -1,386 +0,0 @@ -import { z } from 'zod' -import { McpService, type EndpointTool } from '$lib/gen' -import { createToolDef, type Tool } from '../shared' - -/** - * Generic access to the backend's MCP endpoint catalog (the endpoints marked - * `x-mcp-tool` in openapi.yaml) as three small static tools — search, GET - * call, mutating call — instead of one registered tool per endpoint. Search - * results carry parameter names and usage instructions; full parameter schemas - * enter the model's context only when a call fails validation. This keeps the - * per-iteration tool-schema cost constant. - */ - -// Endpoints whose job a dedicated global tool already does, plus the variable -// read endpoints. Hidden from search and refused at call time: the authoring -// and delete ones would bypass the draft lifecycle (conflict detection, -// explicit deploy, draft cleanup on delete), the variable reads would expose -// variable values to the model (getVariable even decrypts secrets by default), -// and the rest are exact duplicates that would fragment behavior across two -// code paths. -const COVERED_ENDPOINTS: Record = { - getVariable: 'read_workspace_item (variable values are never readable in chat)', - listVariable: 'list_workspace_items (variable values are never readable in chat)', - // The item read/list endpoints return deployed state only, blind to the user's - // drafts; read_workspace_item / list_workspace_items merge drafts, and for - // flows return the compact JSON that patch_flow_json matches against. - getScriptByPath: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - getFlowByPath: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - getResource: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - getSchedule: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - listScripts: 'list_workspace_items (it includes your drafts)', - listFlows: 'list_workspace_items (it includes your drafts)', - listResource: 'list_workspace_items (it includes your drafts)', - listSchedules: 'list_workspace_items (it includes your drafts)', - deleteScriptByPath: 'delete_workspace_item', - deleteScriptByHash: 'delete_workspace_item', - deleteFlowByPath: 'delete_workspace_item', - deleteSchedule: 'delete_workspace_item', - deleteVariable: 'delete_workspace_item', - deleteResource: 'delete_workspace_item', - createScript: 'write_script', - createFlow: 'write_flow', - updateFlow: 'patch_flow_json or write_flow', - createApp: 'init_app and the app draft tools', - updateApp: 'write_app_file / write_app_runnable', - createVariable: 'write_variable', - updateVariable: 'write_variable', - createResource: 'write_resource', - updateResource: 'write_resource', - createSchedule: 'write_schedule', - updateSchedule: 'write_schedule', - searchDocs: 'search_docs', - readDocsPage: 'read_docs_page', - listJobs: 'list_runs', - getJobLogs: 'get_job_logs', - runScriptPreviewAndWaitResult: 'test_run_script' -} - -const MAX_SEARCH_RESULTS = 10 -const MAX_DESCRIPTION_CHARS = 200 -const MAX_INSTRUCTIONS_CHARS = 400 -const MAX_RESULT_CHARS = 20_000 - -let catalogCache: { workspace: string; endpoints: EndpointTool[] } | undefined - -async function loadCatalog(workspace: string): Promise { - if (catalogCache?.workspace !== workspace) { - const endpoints = await McpService.listMcpTools({ workspace }) - catalogCache = { workspace, endpoints } - } - return catalogCache.endpoints -} - -export function clearApiCatalogCache() { - catalogCache = undefined -} - -function tokenize(text: string): string[] { - return text - .replace(/([a-z0-9])([A-Z])/g, '$1 $2') - .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((t) => t.length > 1) -} - -// Cheap plural-insensitive comparison so "workers" matches "worker" and vice versa. -function tokenMatches(token: string, queryToken: string): boolean { - const strip = (t: string) => (t.length > 3 && t.endsWith('s') ? t.slice(0, -1) : t) - return strip(token) === strip(queryToken) -} - -function pathSegments(path: string): string[] { - return path.split('/').filter((seg) => seg && seg !== 'w' && !seg.startsWith('{')) -} - -// Name and path tokens identify the operation; description tokens only support it. -function scoreEndpoint(endpoint: EndpointTool, queryTokens: string[]): number { - const nameTokens = [...tokenize(endpoint.name), ...pathSegments(endpoint.path).flatMap(tokenize)] - const descTokens = tokenize(`${endpoint.description} ${endpoint.instructions}`) - let score = 0 - for (const qt of queryTokens) { - if (nameTokens.some((t) => tokenMatches(t, qt))) score += 3 - else if (descTokens.some((t) => tokenMatches(t, qt))) score += 1 - } - return score -} - -function schemaPropertyNames(schema: unknown): string[] { - const properties = (schema as { properties?: Record } | null | undefined) - ?.properties - return properties ? Object.keys(properties).filter((k) => k !== 'workspace') : [] -} - -function truncate(text: string, max: number): string { - return text.length > max ? text.slice(0, max) + '…' : text -} - -function summarizeEndpoint(endpoint: EndpointTool) { - const params = [ - ...schemaPropertyNames(endpoint.path_params_schema), - ...schemaPropertyNames(endpoint.query_params_schema) - ] - const bodyParams = schemaPropertyNames(endpoint.body_schema) - const instructions = endpoint.instructions.trim() - return { - name: endpoint.name, - endpoint: `${endpoint.method.toUpperCase()} ${endpoint.path}`, - description: truncate(endpoint.description, MAX_DESCRIPTION_CHARS), - ...(instructions ? { instructions: truncate(instructions, MAX_INSTRUCTIONS_CHARS) } : {}), - ...(params.length > 0 ? { params } : {}), - ...(bodyParams.length > 0 ? { body_params: bodyParams } : {}) - } -} - -function endpointSchemaHelp(endpoint: EndpointTool) { - return { - name: endpoint.name, - endpoint: `${endpoint.method.toUpperCase()} ${endpoint.path}`, - path_params_schema: endpoint.path_params_schema ?? undefined, - query_params_schema: endpoint.query_params_schema ?? undefined, - body_schema: endpoint.body_schema ?? undefined - } -} - -async function resolveEndpoint( - workspace: string, - name: string -): Promise<{ endpoint: EndpointTool } | { error: string }> { - const covered = COVERED_ENDPOINTS[name] - if (covered) { - return { error: `"${name}" is covered by the dedicated ${covered} tool — use it instead.` } - } - const catalog = await loadCatalog(workspace) - const endpoint = catalog.find((e) => e.name === name) - if (!endpoint) { - return { - error: `Unknown endpoint "${name}". Use search_api_endpoints to find the endpoint name.` - } - } - return { endpoint } -} - -async function executeEndpoint( - endpoint: EndpointTool, - workspace: string, - params: Record, - body?: unknown -): Promise { - let url = `/api${endpoint.path.replace('{workspace}', encodeURIComponent(workspace))}` - const queryParams = new URLSearchParams() - for (const [key, value] of Object.entries(params)) { - if (value === undefined || value === null) continue - if (url.includes(`{${key}}`)) { - url = url.replace(`{${key}}`, encodeURIComponent(String(value))) - } else { - queryParams.append(key, String(value)) - } - } - const unresolved = [...url.matchAll(/\{([^}]+)\}/g)].map((m) => m[1]) - if (unresolved.length > 0) { - return JSON.stringify({ - success: false, - error: `Missing required path parameter(s): ${unresolved.join(', ')}`, - schema: endpointSchemaHelp(endpoint) - }) - } - const search = queryParams.toString() - if (search) url += `?${search}` - - const method = endpoint.method.toUpperCase() - const fetchOptions: RequestInit = { method } - if (body !== undefined && method !== 'GET') { - fetchOptions.headers = { 'Content-Type': 'application/json' } - fetchOptions.body = JSON.stringify(body) - } - - const response = await fetch(url, fetchOptions) - const raw = response.headers.get('content-type')?.includes('application/json') - ? await response.json() - : await response.text() - if (!response.ok) { - return JSON.stringify({ - success: false, - status: response.status, - error: typeof raw === 'string' ? raw : JSON.stringify(raw), - // 4xx usually means wrong arguments — echo the schema so the model can - // self-correct in the next call without a separate schema-fetch tool. - ...(response.status >= 400 && response.status < 500 - ? { schema: endpointSchemaHelp(endpoint) } - : {}) - }) - } - const result = JSON.stringify({ success: true, data: raw }) - if (result.length <= MAX_RESULT_CHARS) return result - return JSON.stringify({ - success: true, - truncated: true, - data: (typeof raw === 'string' ? raw : JSON.stringify(raw)).slice(0, MAX_RESULT_CHARS), - note: `Result truncated to ${MAX_RESULT_CHARS} characters. Use filter or pagination parameters to narrow it.` - }) -} - -const searchApiEndpointsSchema = z.object({ - query: z - .string() - .describe( - 'Keywords matched against endpoint names, paths, and descriptions (e.g. "workers", "queue", "run flow"). Jobs are called "runs" in the UI.' - ) -}) - -const callApiGetSchema = z.object({ - name: z.string().describe('Endpoint name as returned by search_api_endpoints'), - params: z - .record(z.string(), z.any()) - .optional() - .describe( - 'Path and query parameter values, keyed by parameter name. The workspace parameter is filled automatically.' - ) -}) - -const callApiEndpointSchema = z.object({ - name: z.string().describe('Endpoint name as returned by search_api_endpoints'), - params: z - .record(z.string(), z.any()) - .optional() - .describe( - 'Path and query parameter values, keyed by parameter name. The workspace parameter is filled automatically.' - ), - body: z - .record(z.string(), z.any()) - .optional() - .describe('JSON request body, when the endpoint takes one') -}) - -export const apiCatalogTools: Tool<{}>[] = [ - { - def: createToolDef( - searchApiEndpointsSchema, - 'search_api_endpoints', - 'Search the Windmill REST API endpoint catalog for operations no dedicated tool covers (workers, queue state, job details, running deployed items, deletions, ...). Returns endpoint names to pass to call_api_get or call_api_endpoint.' - ), - planModeSafe: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = searchApiEndpointsSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { content: 'Searching API endpoints...' }) - const catalog = await loadCatalog(workspace) - const queryTokens = tokenize(parsed.query) - const available = catalog.filter((e) => !COVERED_ENDPOINTS[e.name]) - const scored = available - .map((endpoint) => ({ endpoint, score: scoreEndpoint(endpoint, queryTokens) })) - .filter((s) => s.score > 0) - .sort((a, b) => b.score - a.score || a.endpoint.name.localeCompare(b.endpoint.name)) - const coveredHits = catalog - .filter((e) => COVERED_ENDPOINTS[e.name] && scoreEndpoint(e, queryTokens) > 0) - .map((e) => `${e.name} → use ${COVERED_ENDPOINTS[e.name]}`) - - if (scored.length === 0) { - const categories = [...new Set(available.map((e) => pathSegments(e.path)[0]))].sort() - const result = JSON.stringify( - { - matches: [], - hint: `No endpoint matched. Available endpoint categories: ${categories.join(', ')}. Retry with different keywords, or use a dedicated tool if one covers the need.`, - ...(coveredHits.length > 0 ? { covered_by_dedicated_tools: coveredHits } : {}) - }, - null, - 2 - ) - toolCallbacks.setToolStatus(toolId, { content: 'No matching API endpoint', result }) - return result - } - const top = scored.slice(0, MAX_SEARCH_RESULTS) - const result = JSON.stringify( - { - matches: top.map((s) => summarizeEndpoint(s.endpoint)), - ...(scored.length > top.length - ? { - note: `${scored.length - top.length} more match(es) — refine the query to see them.` - } - : {}), - ...(coveredHits.length > 0 ? { covered_by_dedicated_tools: coveredHits } : {}) - }, - null, - 2 - ) - toolCallbacks.setToolStatus(toolId, { - content: `Found ${top.length} API endpoint(s) for "${parsed.query}"`, - result - }) - return result - } - }, - { - def: createToolDef( - callApiGetSchema, - 'call_api_get', - 'Call a read-only GET endpoint from the API catalog by name. Use search_api_endpoints first to find the endpoint name; a failed call returns the parameter schema.' - ), - // Readonly rests on the method check below plus the catalog only exposing - // side-effect-free GETs; never mark a mutating GET as an `x-mcp-tool`. - planModeSafe: true, - showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = callApiGetSchema.parse(args) - const resolved = await resolveEndpoint(workspace, parsed.name) - if ('error' in resolved) { - toolCallbacks.setToolStatus(toolId, { content: resolved.error, error: resolved.error }) - return JSON.stringify({ success: false, error: resolved.error }) - } - if (resolved.endpoint.method.toUpperCase() !== 'GET') { - const error = `"${parsed.name}" is a ${resolved.endpoint.method.toUpperCase()} endpoint — use call_api_endpoint for mutating calls.` - toolCallbacks.setToolStatus(toolId, { content: error, error }) - return JSON.stringify({ success: false, error }) - } - toolCallbacks.setToolStatus(toolId, { content: `Calling ${parsed.name}...` }) - const result = await executeEndpoint(resolved.endpoint, workspace, parsed.params ?? {}) - const ok = JSON.parse(result).success === true - toolCallbacks.setToolStatus(toolId, { - content: ok ? `Called ${parsed.name}` : `Call to ${parsed.name} failed`, - result, - ...(ok ? {} : { error: `Call to ${parsed.name} failed` }) - }) - return result - } - }, - { - def: createToolDef( - callApiEndpointSchema, - 'call_api_endpoint', - 'Call a mutating (POST/PUT/PATCH/DELETE) endpoint from the API catalog by name; the user is asked to confirm. Use search_api_endpoints first to find the endpoint name; a failed call returns the parameter schema.' - ), - requiresConfirmation: true, - confirmationMessage: (args) => `Call API endpoint ${args?.name ?? ''}`, - showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = callApiEndpointSchema.parse(args) - const resolved = await resolveEndpoint(workspace, parsed.name) - if ('error' in resolved) { - toolCallbacks.setToolStatus(toolId, { content: resolved.error, error: resolved.error }) - return JSON.stringify({ success: false, error: resolved.error }) - } - if (resolved.endpoint.method.toUpperCase() === 'GET') { - const error = `"${parsed.name}" is a GET endpoint — use call_api_get (no confirmation needed).` - toolCallbacks.setToolStatus(toolId, { content: error, error }) - return JSON.stringify({ success: false, error }) - } - toolCallbacks.setToolStatus(toolId, { content: `Calling ${parsed.name}...` }) - const result = await executeEndpoint( - resolved.endpoint, - workspace, - parsed.params ?? {}, - parsed.body - ) - const ok = JSON.parse(result).success === true - toolCallbacks.setToolStatus(toolId, { - content: ok ? `Called ${parsed.name}` : `Call to ${parsed.name} failed`, - result, - ...(ok ? {} : { error: `Call to ${parsed.name} failed` }) - }) - return result - } - } -] diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 018c7ffa46..f45af107a2 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -82,12 +82,44 @@ vi.mock('$lib/gen', async () => { runScriptPreview: vi.fn(async () => 'job-script-preview'), runFlowPreview: vi.fn(async () => 'job-flow-preview'), runFlowByPath: vi.fn(async () => 'job-flow-by-path'), + runScriptByPath: vi.fn(async () => 'job-script-by-path'), getJob: vi.fn(async () => ({ type: 'CompletedJob', + id: 'job-123', + job_kind: 'script', + script_path: 'f/team/runner', success: true, + canceled: false, + args: { n: 3 }, result: { ok: true }, logs: 'test logs' })), + getJobArgs: vi.fn(async () => ({ big: 'real args' })), + getCompletedJobResultMaybe: vi.fn(async () => ({ completed: true, result: { big: 'x' } })), + getFlowAllResults: vi.fn(async () => ({ + entries: [ + { + job_id: 'job-123', + label: 'Flow', + kind: 'script', + depth: 0, + sibling_index: 1, + sibling_count: 1, + status: 'success', + success: true + } + ], + truncated: false, + scope_filtered: false + })), + // What every job wait polls first; unmocked it reaches the real client and the + // wait never returns. Answers completed, so one tick settles the job. + getJobUpdates: vi.fn(async () => ({ + completed: true, + running: false, + new_logs: 'test logs', + log_offset: 'test logs'.length + })), getJobLogs: vi.fn(async () => 'job log line 1\njob log line 2'), listJobs: vi.fn(async () => [ { @@ -210,7 +242,11 @@ vi.mock('$lib/gen', async () => { }), createResource: vi.fn(async () => 'created'), updateResource: vi.fn(async () => 'updated'), - deleteResource: vi.fn(async () => 'deleted') + deleteResource: vi.fn(async () => 'deleted'), + // A workspace with no skills, which is what makes `read_skill` refuse a path + // the model composed: the gate is membership in the listing. + listResource: vi.fn(async () => []), + getResourceValue: vi.fn(async () => ({ content: 'skill body' })) }), VariableService: wrapService(actual.VariableService, { existsVariable: vi.fn(async () => false), @@ -220,6 +256,14 @@ vi.mock('$lib/gen', async () => { createVariable: vi.fn(async () => 'created'), updateVariable: vi.fn(async () => 'updated') }), + WorkerService: wrapService(actual.WorkerService, { + listWorkers: vi.fn(async () => []) + }), + // Guests off by default, as a fresh workspace has them. + WorkspaceService: wrapService(actual.WorkspaceService, { + getGuestUsage: vi.fn(async () => ({ available: false, instance_enabled: false })), + getPublicSettings: vi.fn(async () => ({ guest_access_enabled: false })) + }), FolderService: wrapService(actual.FolderService, { createFolder: vi.fn(async () => 'created') }), @@ -228,6 +272,17 @@ vi.mock('$lib/gen', async () => { const user = whoamiByWorkspace.get(workspace) if (!user) throw new Error(`not a member of ${workspace}`) return user + }), + // `refreshSuperadmin` cancels the previous in-flight call, so this stands in for + // the CancelablePromise the real client returns. + globalWhoami: vi.fn(() => { + const pending: any = Promise.resolve({ + email: 'devops@windmill.dev', + super_admin: false, + devops: true + }) + pending.cancel = () => {} + return pending }) }), DraftService: wrapService(actual.DraftService, { @@ -293,6 +348,12 @@ vi.mock('$lib/gen', async () => { } }) +// Minting reaches the API and is covered in secretArgUtils.test.ts; what matters here is that a +// run the posture answers goes through it and starts on what came back. +vi.mock('$lib/components/secretArgUtils', () => ({ + processSecretArgs: vi.fn(async (args: Record) => args) +})) + vi.mock('./rawAppBundlerBridge', () => ({ bundleRawAppDraft: vi.fn(async () => ({ js: 'bundled js', @@ -302,11 +363,13 @@ vi.mock('./rawAppBundlerBridge', () => ({ vi.mock('$lib/infer', async () => ({ ...(await vi.importActual('$lib/infer')), - // Avoid the wasm parser in unit tests: the script deploy path infers the arg - // schema but tolerates failure, and these tests don't assert on the schema. + // Avoid the wasm parser in unit tests. A no-op passes the seeded schema through + // untouched, which is what lets the password-marking test pin how a schema is + // seeded in and carried out without pinning inference's own merge rules. inferArgs: vi.fn(async () => {}) })) +import { inferArgs } from '$lib/infer' import { buildRunsFilterSearchbarSchema } from '$lib/components/runs/runsFilter' import { buildOpenPageUrl, @@ -345,9 +408,12 @@ import { ScheduleService, ScriptService, UserService, - VariableService + VariableService, + WorkerService, + WorkspaceService } from '$lib/gen' -import { superadmin, userStore, usersWorkspaceStore } from '$lib/stores' +import { devopsRole, superadmin, userStore, usersWorkspaceStore } from '$lib/stores' +import { processSecretArgs } from '$lib/components/secretArgUtils' import { clearWorkspaceRoleCache } from '$lib/user' import { get } from 'svelte/store' import type { Tool, ToolCallbacks } from '../shared' @@ -365,9 +431,22 @@ function getBackendDraft(kind: string, path: string, _opts?: unknown): return backendDrafts.get(`${kind}:${path}`) as V | undefined } +// inferArgs is stubbed module-wide (no wasm parser here), so a test whose form is built by +// inference has to say what the next call finds. Once, so a test that also calls write_script +// — which infers to fill the draft's schema — queues this after that write, not before. +function stubInferredProperties(properties: Record): void { + vi.mocked(inferArgs).mockImplementationOnce(async (_lang, _code, schema) => { + schema.properties = properties + return null + }) +} + const toolCallbacks: ToolCallbacks = { setToolStatus: vi.fn(), - removeToolStatus: vi.fn() + removeToolStatus: vi.fn(), + // Every host that can run a script mounts the form, so the default answers it with what it + // opened with. A test meaning to exercise a host without one overrides this with undefined. + requestRunArgs: async (_toolId, form) => form.args } function getGlobalTool(name: string): Tool<{}> { @@ -454,7 +533,7 @@ describe('global AI tools', () => { expect(names).toContain('test_run_script') expect(names).toContain('test_run_flow') expect(names).toContain('test_run_step') - expect(names).toContain('get_job_logs') + expect(names).toContain('get_run') expect(names).toContain('list_runs') }) @@ -680,29 +759,367 @@ describe('global AI tools', () => { ) }) - it('fetches job logs by id and always suppresses the backend ansi hint line', async () => { - const result = await callGlobalTool('get_job_logs', { id: 'job-123' }) + it('lists workers with the diagnostic fields only', async () => { + vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([ + { + worker: 'wk-1', + worker_instance: 'host-1', + worker_group: 'gpu', + custom_tags: ['gpu'], + last_ping: 3, + jobs_executed: 12, + started_at: '2024-01-01T00:00:00Z', + ip: '10.0.0.1', + wm_version: 'v1', + memory: 123, + occupancy_rate: 0.5 + } + ]) + + const result = await callGlobalTool('list_workers', {}) + + expect(JSON.parse(result).workers).toEqual([ + { + worker: 'wk-1', + worker_group: 'gpu', + custom_tags: ['gpu'], + last_ping: 3, + jobs_executed: 12 + } + ]) + // Page telemetry must stay out of the model's context. + expect(result).not.toContain('occupancy_rate') + expect(result).not.toContain('10.0.0.1') + }) + + it('says so when the worker page is cut short', async () => { + vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce( + Array(100).fill({ + worker: 'wk', + worker_group: 'default', + custom_tags: [], + last_ping: 1, + jobs_executed: 0 + }) as any + ) + + const result = await callGlobalTool('list_workers', {}) + + // A full page is indistinguishable from the whole fleet, and the model reasons + // about tag coverage from this list. + expect(JSON.parse(result).note).toContain('Only the first 100 workers') + }) + + describe('list_workers with nothing to show', () => { + afterEach(() => { + superadmin.set(undefined) + devopsRole.set(undefined) + }) + + it('never reports an empty list as an absence to a caller workers can be hidden from', async () => { + superadmin.set(false) + devopsRole.set(false) + vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([]) + + const result = await callGlobalTool('list_workers', {}) + + // An instance hiding workers from a non-devops caller answers with an empty + // list, so absence is unprovable here. + expect(result).toContain('does NOT establish that no workers are running') + expect(result).toContain('devops role') + expect(result).not.toContain('"workers"') + }) + + it('reports an empty list as an absence to a devops caller', async () => { + devopsRole.set('devops@windmill.dev') + vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([]) + + const result = await callGlobalTool('list_workers', {}) + + // Nothing is hidden from this caller, so hedging would withhold the answer a + // stuck queue is waiting on. + expect(result).toContain('No workers are connected') + expect(result).not.toContain('does NOT establish') + }) + + it('resolves the role before deciding, rather than reading unloaded stores as no role', async () => { + // Both stores start undefined; without the refresh a devops caller whose whoami + // has not landed yet is hedged at instead of answered. + expect(get(superadmin)).toBeUndefined() + expect(get(devopsRole)).toBeUndefined() + vi.mocked(WorkerService.listWorkers).mockResolvedValueOnce([]) + + const result = await callGlobalTool('list_workers', {}) + + expect(result).toContain('No workers are connected') + }) + }) + + it('returns args, result and logs of a run in one call', async () => { + const result = await callGlobalTool('get_run', { id: 'job-123' }) expect(JobService.getJobLogs).toHaveBeenCalledWith({ workspace: WORKSPACE, id: 'job-123', + // The backend's "to remove ansi colors, use: sed ..." hint is noise for + // the model, so it is always suppressed. removeAnsiWarnings: true }) - expect(result).toBe('job log line 1\njob log line 2') - // The logs must be surfaced as the tool result so the details panel shows - // them rather than "No result yet". + const parsed = JSON.parse(result) + expect(parsed.run).toMatchObject({ + status: 'success', + path: 'f/team/runner', + args: expect.stringContaining('"n": 3'), + result: expect.stringContaining('"ok": true'), + logs: 'job log line 1\njob log line 2' + }) + // The result must be surfaced as the tool result so the details panel shows + // it rather than "No result yet". expect(toolCallbacks.setToolStatus).toHaveBeenCalledWith( - 'test-get_job_logs', - expect.objectContaining({ result: 'job log line 1\njob log line 2' }) + 'test-get_run', + expect.objectContaining({ result }) ) }) - it('reports when a job has no logs', async () => { + it('names the job the card renders, without changing what the model is handed', async () => { + const runResult = await callGlobalTool('get_run', { id: 'job-123' }) + expect(toolCallbacks.setToolStatus).toHaveBeenLastCalledWith( + 'test-get_run', + expect.objectContaining({ + result: runResult, + inspectedRun: { + jobId: 'job-123', + workspace: WORKSPACE, + runId: 'job-123', + step: undefined + } + }) + ) + + // A step is a job of its own, and the model's line of prose about it carries + // neither its arguments nor its logs — the card reads those from the job. The + // address travels with it, since a step job names neither the step nor its run. + vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ + entries: [ + { + job_id: 'step-job-1', + label: 'b', + kind: 'script', + depth: 1, + sibling_index: 1, + sibling_count: 1, + status: 'success', + success: true, + result_prefix: '{"ok":true}' + } + ] + } as any) + const stepResult = await callGlobalTool('get_run', { id: 'job-123', step: 'b' }) + expect(stepResult).toContain('(job step-job-1, success) result:') + expect(toolCallbacks.setToolStatus).toHaveBeenLastCalledWith( + 'test-get_run', + expect.objectContaining({ + result: stepResult, + inspectedRun: { + jobId: 'step-job-1', + workspace: WORKSPACE, + runId: 'job-123', + step: 'b' + } + }) + ) + + // An address naming several jobs resolves to none of them, so there is + // nothing for the card to bind to and the call renders as an ordinary row. + vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ + entries: [], + step_error: 'Step "b" ran 4 times (loop/branches) — pick one with "b[i]".' + } as any) + await callGlobalTool('get_run', { id: 'job-123', step: 'b' }) + expect(toolCallbacks.setToolStatus).toHaveBeenLastCalledWith( + 'test-get_run', + expect.not.objectContaining({ inspectedRun: expect.anything() }) + ) + }) + + it('reports when a run has no logs, and tells that apart from logs it could not read', async () => { vi.mocked(JobService.getJobLogs).mockResolvedValueOnce(' ') + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-empty' })).run.logs).toBe( + 'No logs for this run.' + ) - const result = await callGlobalTool('get_job_logs', { id: 'job-empty' }) + // A failed fetch must not read as "this run logged nothing" — the model + // would report that to the user as fact. + vi.mocked(JobService.getJobLogs).mockRejectedValueOnce(new Error('boom')) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })).run.logs).toBe( + 'Logs could not be read for this run.' + ) - expect(result).toBe('No logs available for this job.') + // Nor does every failed read reject: the generated client resolves undefined + // when it cannot read the body, which lands on the same "no logs" branch. + vi.mocked(JobService.getJobLogs).mockResolvedValueOnce(undefined as any) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })).run.logs).toBe( + 'Logs could not be read for this run.' + ) + }) + + it('keeps the end of a long log, and never opens it on half a surrogate pair', async () => { + // 12002 code points over 24003 UTF-16 units: the 12000-unit tail opens one + // unit into a unicorn, so the lone low surrogate has to be dropped. + vi.mocked(JobService.getJobLogs).mockResolvedValueOnce('🦄'.repeat(12001) + 'z') + + const [note, body] = JSON.parse( + await callGlobalTool('get_run', { id: 'job-123' }) + ).run.logs.split('\n') + + // The note goes first: the tail is what the model came for, and a note at + // the end would read as the last thing the run logged. + expect(note).toContain('12002 chars total') + expect(body).toHaveLength(11999) + expect(body.codePointAt(0)).toBe(0x1f984) + expect(body.endsWith('🦄z')).toBe(true) + }) + + it('keeps the step tree optional: a failed tree fetch still returns the run itself', async () => { + vi.mocked(JobService.getFlowAllResults).mockRejectedValueOnce(new Error('tree unavailable')) + + const parsed = JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })) + + expect(parsed.run).toMatchObject({ args: expect.stringContaining('"n": 3') }) + expect(parsed.run.logs).toBe('job log line 1\njob log line 2') + // Silence here would read as "this flow ran no steps", which the model + // would then report to the user as fact. + expect(parsed.run.steps_unavailable).toBe(true) + // The tree's root entry normally names the run; without it nothing does. + expect(parsed.run.job_id).toBe('job-123') + + // And the tree read fails the same two ways the log read does: reading + // `.entries` off a resolved undefined throws, which would cost the model the + // job and logs already in hand rather than just the tree. + vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce(undefined as any) + const noTree = JSON.parse(await callGlobalTool('get_run', { id: 'job-123' })) + expect(noTree.run.steps_unavailable).toBe(true) + expect(noTree.run.logs).toBe('job log line 1\njob log line 2') + }) + + it('reports why a run was canceled or died, the fields getJob used to carry', async () => { + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-oom', + job_kind: 'script', + success: false, + canceled: true, + canceled_by: 'alice', + canceled_reason: 'exceeded memory limit', + mem_peak: 2097152 + } as any) + + const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-oom' })).run + + expect(run.status).toBe('canceled') + expect(run.canceled_by).toBe('alice') + expect(run.canceled_reason).toBe('exceeded memory limit') + expect(run.mem_peak_kb).toBe(2097152) + }) + + // Over ~90KB the job endpoint elides the payload, leaving the step tree as the + // only place a real (server-side truncated) head of the result survives. + function mockElidedJob() { + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-big', + job_kind: 'script', + success: true, + canceled: false, + args: { reason: 'WINDMILL_TOO_BIG' }, + result: 'WINDMILL_TOO_BIG' + } as any) + vi.mocked(JobService.getFlowAllResults).mockResolvedValueOnce({ + entries: [ + { + job_id: 'job-big', + label: 'Flow', + kind: 'script', + depth: 0, + sibling_index: 1, + sibling_count: 1, + status: 'success', + success: true, + result_prefix: '{"blob":"real head"}', + result_length: 300018 + } + ], + truncated: false, + scope_filtered: false + } as any) + } + + it("reports getJob's WINDMILL_TOO_BIG placeholders instead of fetching around them", async () => { + mockElidedJob() + + const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-big' })).run + + // The marker is never the run's own value, so it must not reach the model. + expect(JSON.stringify(run)).not.toContain('WINDMILL_TOO_BIG') + // The result falls to the step tree's real server-side head, flagged and + // sized so the model can't mistake the fragment for the whole payload. + expect(run.result).toBe('{"blob":"real head"}') + expect(run.result_total_chars).toBe(300018) + expect(run.result_truncated).toBe(true) + expect(run.args_truncated).toBe(true) + expect(run.args).toBeUndefined() + // The endpoints that return these whole take no length parameter, so + // reaching for one would pull the entire payload into the tab. + expect(JobService.getJobArgs).not.toHaveBeenCalled() + expect(JobService.getCompletedJobResultMaybe).not.toHaveBeenCalled() + }) + + it('keeps a payload that merely carries the marker string in a reason of its own', async () => { + // The backend elides a result to the bare string and args to exactly + // {reason: marker}. A payload with that reason plus fields of its own is the + // run's own value, and withholding it would report an elision that never was. + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-reason', + job_kind: 'script', + success: false, + canceled: false, + args: { reason: 'WINDMILL_TOO_BIG', retries: 2 }, + result: { reason: 'WINDMILL_TOO_BIG', code: 42 } + } as any) + + const run = JSON.parse(await callGlobalTool('get_run', { id: 'job-reason' })).run + + expect(run.args_truncated).toBeUndefined() + expect(run.result_truncated).toBeUndefined() + expect(run.args).toContain('"retries": 2') + expect(run.result).toContain('"code": 42') + }) + + it('reports skipped and suspended runs as such rather than success or running', async () => { + // `success` is true for a skipped job, and a suspended job is `running`. + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'CompletedJob', + id: 'job-skipped', + job_kind: 'script', + success: true, + canceled: false, + is_skipped: true + } as any) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-skipped' })).run.status).toBe( + 'skipped' + ) + + vi.mocked(JobService.getJob).mockResolvedValueOnce({ + type: 'QueuedJob', + id: 'job-suspended', + job_kind: 'flow', + running: true, + suspend: 1 + } as any) + expect(JSON.parse(await callGlobalTool('get_run', { id: 'job-suspended' })).run.status).toBe( + 'suspended' + ) }) it('searches hub scripts without fetching script contents', async () => { @@ -1643,6 +2060,69 @@ describe('global AI tools', () => { }) }) + it('tells a code app from a drag-and-drop app', async () => { + vi.mocked(AppService.listApps).mockResolvedValueOnce([ + { path: 'f/apps/code', summary: 'Code app', raw_app: true }, + { path: 'f/apps/builder', summary: 'Builder app' } + ] as any) + // An app draft is always a code app: the chat cannot address a + // drag-and-drop app's draft kind at all. + seedBackendDraft( + 'raw_app', + 'u/admin/draft_listed', + { summary: 'Draft app' }, + { workspace: WORKSPACE } + ) + + const rows = JSON.parse(await callGlobalTool('list_workspace_items', { types: ['app'] })) + + expect(rows.map((r: any) => [r.path, r.rawApp])).toEqual([ + ['f/apps/code', true], + ['f/apps/builder', false], + ['u/admin/draft_listed', true] + ]) + }) + + it('still says which kind of app it is when the app is read directly', async () => { + // The flag decides whether the app tools are offered at all, and the model + // reads an app before it edits one — a listing that knows is not enough. + vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ + path: 'f/apps/builder', + summary: 'Builder app', + value: { grid: [] }, + raw_app: false + } as any) + + const read = JSON.parse( + await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/builder' }) + ) + expect(read.rawApp).toBe(false) + }) + + it('finds a staged app under the folder it was filed in, not its generated path', async () => { + // A never-deployed app the editor created lives at a generated path, so the + // folder the user filed it under exists only as its staged name. The server + // drops draft-only rows under any narrowing filter, leaving this pass the one + // that can answer a folder-scoped question about it. + seedBackendDraft( + 'raw_app', + 'u/admin/draft_7f21c9', + { summary: '', draft_path: 'f/team/invoice_tracker' }, + { workspace: WORKSPACE } + ) + + const matched = JSON.parse( + await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/team/' }) + ) + expect(matched).toHaveLength(1) + expect(matched[0].draftPath).toBe('f/team/invoice_tracker') + + const other = JSON.parse( + await callGlobalTool('list_workspace_items', { types: ['app'], path_prefix: 'f/other/' }) + ) + expect(other).toEqual([]) + }) + it('applies path_prefix to drafts before enforcing the result limit', async () => { await callGlobalTool('write_script', { path: 'f/other/outside', @@ -3152,6 +3632,26 @@ describe('global AI tools', () => { }) }) + // The draft carries the deployed policy from the fork, so nothing is fetched to answer. + it('reports exposure for an app that has a draft over it', async () => { + seedBackendDraft( + 'raw_app', + 'f/apps/drafted', + { + files: { '/src/App.tsx': 'x' }, + runnables: {}, + policy: { execution_mode: 'anonymous' } + } as any, + { workspace: WORKSPACE } + ) + + const read = await callGlobalTool('read_workspace_item', { + type: 'app', + path: 'f/apps/drafted' + }) + expect(JSON.parse(read)).toMatchObject({ isDraft: true, executionMode: 'anonymous' }) + }) + it('summarizes local raw app drafts in read_workspace_item', async () => { seedBackendDraft( 'raw_app', @@ -3267,6 +3767,90 @@ describe('global AI tools', () => { expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() }) + // A low-code app has a grid, not files and runnables, so every app tool here would + // otherwise report it as empty rather than say it is the wrong kind of app. + it('reports a low-code app without its contents, and refuses to act on it', async () => { + const lowCode = { + path: 'f/apps/legacy', + summary: 'legacy app', + raw_app: false, + value: { grid: [{ id: 'a', data: { type: 'buttoncomponent' } }] } + } as any + // ...Once per call: a persistent implementation would outlive this test and + // disarm the factory's "mock not configured" guard for the rest of the file. + for (let i = 0; i < 3; i++) vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(lowCode) + + // The read answers "this app is drag-and-drop" rather than throwing — but it must not + // summarize a grid as a file/runnable list, which reads as an empty app. + const read = JSON.parse( + await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/legacy' }) + ) + expect(read).toMatchObject({ type: 'app', path: 'f/apps/legacy', rawApp: false }) + expect(read).not.toHaveProperty('value') + + // The tools that would convert it to files and runnables still refuse: a staged draft + // would answer every later read in place of the app itself. + for (const [tool, args] of [ + ['read_app_file', { path: 'f/apps/legacy', file_path: '/index.tsx' }], + ['write_app_file', { path: 'f/apps/legacy', file_path: '/App.tsx', content: 'x' }] + ] as [string, any][]) { + const raw = await callGlobalTool(tool, args).catch((e) => String(e)) + expect(raw).toContain('low-code app') + } + expect(getBackendDraft('raw_app', 'f/apps/legacy', { workspace: WORKSPACE })).toBeUndefined() + }) + + // This read is the only way to ask who may open a deployed app. Both kinds answer: a + // drag-and-drop app can be anonymous too, and no other tool here can inspect one. + it('reports who may open an app, whichever kind it is', async () => { + const readMode = async (app: any) => { + vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(app) + const read = await callGlobalTool('read_workspace_item', { type: 'app', path: app.path }) + return JSON.parse(read).executionMode + } + const guestApp = { + path: 'f/apps/code', + summary: 'Code app', + raw_app: true, + value: { files: {}, runnables: {} }, + policy: { execution_mode: 'guest' } + } + + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + expect(await readMode(guestApp)).toBe('guest') + + expect( + await readMode({ + path: 'f/apps/builder', + summary: 'Builder app', + raw_app: false, + value: { grid: [] }, + policy: { execution_mode: 'anonymous' } + }) + ).toBe('anonymous') + + // The same app, with each switch crossed in turn: either one alone admits nobody, so + // reporting the mode bare would name an exposure the server refuses. Both off needs + // no case of its own — whichever half of the check were dropped, one of these two + // still catches it. + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + expect(await readMode(guestApp)).toContain('inert') + + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + expect(await readMode(guestApp)).toContain('inert') + }) + it('reads raw app files without creating a draft', async () => { vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ path: 'f/apps/report', @@ -3923,6 +4507,69 @@ describe('global AI tools', () => { expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() }) + // Deploying is what makes an app's runnables reachable, so it is the one moment the + // exposure is both true and known. + it('discloses who can open an app after a deploy', async () => { + const deployApp = async (path: string, policy: Record) => { + vi.mocked(AppService.existsApp).mockResolvedValueOnce(true) + vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({} as any) + seedBackendDraft( + 'raw_app', + path, + { + summary: 'App', + files: { '/index.tsx': 'x' }, + runnables: {}, + data: { tables: [] }, + policy + }, + { workspace: WORKSPACE } + ) + return JSON.parse(await callGlobalTool('deploy_workspace_item', { type: 'app', path })) + } + + const open = await deployApp('f/apps/open', { + execution_mode: 'anonymous', + on_behalf_of: 'u/alice' + }) + expect(open.success).toBe(true) + expect(open.message).toContain('anyone with the URL, without logging in') + // The server decides what a deployed app runs as — it overwrites on_behalf_of for + // anyone outside the deployers group — so the note must not name an identity. + expect(open.message).not.toContain('u/alice') + + // Guest is a real widening only where the deployment, the instance and the + // workspace all admit guests; stored below that, it is inert. + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + const guestOn = await deployApp('f/apps/guest', { execution_mode: 'guest' }) + expect(guestOn.success).toBe(true) + expect(guestOn.message).toContain('identity provider authenticates') + + // Each switch crossed in turn, because either one alone admits nobody and the note + // would then announce an exposure that does not exist. Both off needs no case of its + // own: whichever half of the check were dropped, one of these two still catches it. + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + const instanceOff = await deployApp('f/apps/guest_inst_off', { execution_mode: 'guest' }) + expect(instanceOff.success).toBe(true) + expect(instanceOff.message).not.toContain('identity provider') + + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + const workspaceOff = await deployApp('f/apps/guest_ws_off', { execution_mode: 'guest' }) + expect(workspaceOff.success).toBe(true) + expect(workspaceOff.message).not.toContain('identity provider') + }) + it('forwards preserve_on_behalf_of when the deployed policy carries an on_behalf_of', async () => { // Without the flag the backend resets the policy's on_behalf_of to the // deploying user; this chat path has no on-behalf-of selector, so it must @@ -4353,8 +5000,7 @@ describe('global AI tools', () => { const result = await withCompletedTestJob(() => callGlobalTool('test_run_script', { - path: 'f/scripts/draft-test', - args: { name: 'Ada' } + path: 'f/scripts/draft-test' }) ) @@ -4363,7 +5009,7 @@ describe('global AI tools', () => { requestBody: { path: 'f/scripts/draft-test', content, - args: { name: 'Ada' }, + args: {}, language: 'bun' } }) @@ -4372,6 +5018,43 @@ describe('global AI tools', () => { expect(result).toContain('test logs') }) + // No parser emits `password`, so the stored schema is the only thing carrying it: an edit + // that rewrites the draft's schema from scratch, or a draft read that drops it, unmarks + // the field — and the form then takes the secret as a plain literal into the job's args. + it('test_run_script keeps the password marking of the script it previews', async () => { + vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(true) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/secretful', + language: 'bun', + schema: { + type: 'object', + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + } as any) + + await callGlobalTool('write_script', { + path: 'f/scripts/secretful', + language: 'bun', + content: 'export async function main(token: string) { return 1 }' + }) + + let form: any + await callGlobalTool( + 'test_run_script', + { path: 'f/scripts/secretful' }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, opened) => { + form = opened + return undefined + } + } + ) + + expect(form?.schema?.properties).toMatchObject({ token: { password: true } }) + }) + it('test_run_script previews deployed script content when no draft exists', async () => { vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ path: 'f/scripts/deployed-test', @@ -4382,8 +5065,7 @@ describe('global AI tools', () => { await withCompletedTestJob(() => callGlobalTool('test_run_script', { - path: 'f/scripts/deployed-test', - args: { name: 'Grace' } + path: 'f/scripts/deployed-test' }) ) @@ -4396,18 +5078,412 @@ describe('global AI tools', () => { requestBody: { path: 'f/scripts/deployed-test', content: 'def main(name):\n return name', - args: { name: 'Grace' }, + args: {}, language: 'python3' } }) }) + // A test run meets the same card as a deployed one, so what previews is what the form + // submitted — not what the model proposed. + it('test_run_script previews the arguments the form submitted', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/formed-test', + summary: 'Formed test script', + content: 'export async function main(name: string) {}', + language: 'bun', + schema: { properties: { name: { type: 'string' } } } + } as any) + + let opened: Record | undefined + let kind: string | undefined + let helperSource: { code?: string; lang?: string } | undefined + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_script', + { path: 'f/scripts/formed-test', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + opened = form.args + kind = form.kind + helperSource = { code: form.code, lang: form.lang } + return { name: 'Grace' } + } + } + ) + ) + + expect(opened).toEqual({ name: 'Ada' }) + // Drives the card's tense: a test says it tested, not that it ran. + expect(kind).toBe('test') + // The draft itself, so a dynselect field offers the options this code returns rather + // than the deployed version's — which is stale, or absent for a draft never deployed. + expect(helperSource).toEqual({ + code: 'export async function main(name: string) {}', + lang: 'bun' + }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/scripts/formed-test', + content: 'export async function main(name: string) {}', + args: { name: 'Grace' }, + language: 'bun' + } + }) + }) + + // The bypass posture answers a run form as it answers any other confirmation, for a + // deployed run as much as a test. The card must never render one first: a form nobody + // will fill in is attached already settled, so no field is ever mounted, and the schema + // it would have built them from never reaches the transcript. + it('mounts no field on either run form under yolo', async () => { + const script = { + path: 'f/scripts/yolo', + content: 'export async function main(name: string) {}', + language: 'bun', + schema: { properties: { name: { type: 'string' } } } + } as any + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce(script) + + const statuses: any[] = [] + const requestRunArgs = vi.fn(async (_toolId: string, form: any) => form.args) + const yolo = { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs + } + + await withCompletedTestJob(() => + callGlobalTool('test_run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) + ) + + const testForm = statuses.find((s) => s.runForm)?.runForm + expect(testForm.submitted).toBe(true) + // Nothing is left to render it, and a card carrying one persists it forever. + expect(testForm.schema).toBeUndefined() + // Told the form is already answered, or the loop parks on a card with no fields. + expect(requestRunArgs.mock.calls[0][2]).toEqual({ autoAccepted: true }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: expect.objectContaining({ args: { name: 'Ada' } }) }) + ) + + statuses.length = 0 + await withCompletedTestJob(() => + callGlobalTool('run_script', { path: 'f/scripts/yolo', args: { name: 'Ada' } }, yolo) + ) + + const deployedForm = statuses.find((s) => s.runForm)?.runForm + expect(deployedForm.submitted).toBe(true) + expect(deployedForm.schema).toBeUndefined() + // skipPreprocessor: the form fills the main input schema, so a preprocessor would take + // these arguments for a webhook body and run main on its output instead. + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' }, skipPreprocessor: true }) + ) + }) + + // The posture is the user's standing answer to whether to ask, so a run it answers starts on + // the model's arguments as sent — no default filled in, no required field second-guessed. + // Predicting what a mounted field would hold starts runs the form itself would refuse; the + // schema's own defaults are the worker's job, from the code's signature. + it('sends the model arguments as proposed when the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/defaulted', + schema: { + properties: { name: { type: 'string' }, retries: { type: 'number', default: 3 } }, + required: ['name', 'retries'] + } + } as any) + const statuses: any[] = [] + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/defaulted', args: { name: 'Ada' } }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(statuses.find((x) => x.runForm)?.runForm.submitted).toBe(true) + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' } }) + ) + }) + + // A disabled field is declared as not the caller's to set, and the posture answering the + // form does not make the model one of the callers it is kept from. + it('holds a disabled default against the model when the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/locked', + schema: { + properties: { + name: { type: 'string' }, + mode: { type: 'string', disabled: true, default: 'safe' } + } + } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/locked', args: { name: 'Ada', mode: 'destructive' } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada', mode: 'safe' } }) + ) + // Or the next call proposes the same override again. + expect(result).toContain('disables mode') + }) + + // With no form there is no PasswordArgInput to turn a proposed secret into a reference, and + // a job's arguments are readable by everyone who can see its run. What starts the job must + // be what came back from the minting, never the proposal. + it('mints a proposed secret into a reference before starting a run the posture answers', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/secret', + schema: { properties: { token: { type: 'string', password: true } }, required: ['token'] } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/secret', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(processSecretArgs).toHaveBeenCalledWith( + { token: 'hunter2' }, + expect.anything(), + expect.anything() + ) + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The bypass is the user's standing answer, not a licence for the host to skip asking: + // a chat with nowhere to put a form still refuses the run under any other posture. + it('run_script refuses a host with no form unless the posture answers for it', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noform', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const refused = await callGlobalTool( + 'run_script', + { path: 'f/scripts/noform', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: undefined } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(refused).toContain('cannot show a run form') + }) + + // The posture answers wherever it is set, form or no form: what it answers is consent, and a + // host without one has nothing left to ask. A secret still becomes a reference first, which + // is the only thing the missing form would have done. + it('run_script runs on a formless host under yolo', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noform-secret', + schema: { + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/noform-secret', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + requestRunArgs: undefined, + shouldAutoAcceptToolConfirmations: () => true + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The transcript is re-cloned into IndexedDB on every save, and a form takes as much text + // as the user pastes. What the card stores is bounded; what the job runs is not. + it('run_script stores a marker for oversized arguments but runs them in full', async () => { + const huge = 'x'.repeat(120_000) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/big', + content: 'export async function main(blob: string) {}', + language: 'bun', + schema: { properties: { blob: { type: 'string' } } } + } as any) + + const statuses: any[] = [] + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/big', args: { blob: 'small' } }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + // The user pastes into the field the model left small: the model's own + // proposal is bounded by what it can emit, this is not. + requestRunArgs: async () => ({ blob: huge }) + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { blob: huge } }) + ) + const persisted = statuses.filter((s) => s.parameters !== undefined).at(-1)?.parameters + expect(persisted).toEqual({ reason: 'WINDMILL_TOO_BIG' }) + expect(JSON.stringify(statuses)).not.toContain(huge) + }) + + // A dropped pass-through fails silently: the argument is accepted and ignored, and the + // run just waits out the default budget. + it('run_script detaches on background and on wait_seconds', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/slow', + schema: { properties: { name: { type: 'string' } } } + } as any) + + // wait_seconds 0 detaches the same way, so one run of each pins both lines. + for (const detachArg of [{ background: true }, { wait_seconds: 0 }]) { + const onJobDetached = vi.fn() + await callGlobalTool( + 'run_script', + { path: 'f/scripts/slow', args: { name: 'Ada' }, ...detachArg }, + { ...toolCallbacks, onJobStarted: vi.fn(), onJobDetached } + ) + + expect(onJobDetached).toHaveBeenCalledWith('job-script-by-path') + } + }) + + // A schema with no fields still opens a form: an empty one is still the Run button, and + // that button is the whole confirmation this tool has. Skipping it because there is + // nothing to fill in starts the script with no confirmation at all. + it('test_run_script opens a form and starts no job when the schema declares no field', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/noargs-test', + content: 'export async function main() {}', + language: 'bun', + schema: { properties: {} } + } as any) + + const cancelled = await callGlobalTool( + 'test_run_script', + { path: 'f/scripts/noargs-test', args: { force_delete: true } }, + { ...toolCallbacks, requestRunArgs: async () => undefined } + ) + + expect(JobService.runScriptPreview).not.toHaveBeenCalled() + expect(cancelled).toContain('The user cancelled the run form') + + // A schema declaring nothing is a form with no field to hold this, and the card says + // as much — so answering it must not send an argument that was never on screen. + const answered = await withCompletedTestJob(() => + callGlobalTool( + 'test_run_script', + { path: 'f/scripts/noargs-test', args: { force_delete: true } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(JobService.runScriptPreview).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: expect.objectContaining({ args: {} }) }) + ) + expect(answered).toContain('does not declare force_delete') + }) + + // A host that answers the form without minting, as the eval harness does by returning the + // proposal verbatim: the job's arguments are readable by everyone who can see its run. + it('mints a secret the host handed back as a literal', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/host-literal', + schema: { properties: { token: { type: 'string', password: true } } } + } as any) + vi.mocked(processSecretArgs).mockImplementationOnce(async () => ({ + token: '$var:u/ada/secret_arg/minted' + })) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/host-literal', args: { token: 'hunter2' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { token: '$var:u/ada/secret_arg/minted' } }) + ) + }) + + // The bypass has no form to have shown them either, so the rule holds there too. + it('drops an undeclared argument when the posture answers too', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/noargs-yolo', + schema: { properties: { name: { type: 'string' } } } + } as any) + + await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/noargs-yolo', args: { name: 'Ada', force_delete: true } }, + { + ...toolCallbacks, + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith( + expect.objectContaining({ requestBody: { name: 'Ada' } }) + ) + }) + + // The form offers the arguments the flow declares and no others, so a fixture flow that + // takes one has to say so — as a real flow does, since nothing else could render a field. + const FLOW_NAME_SCHEMA = { type: 'object', properties: { name: { type: 'string' } } } + it('test_run_flow previews draft flow content by path', async () => { const modules = [{ id: 'start', value: { type: 'identity' } }] await callGlobalTool('write_flow', { path: 'f/flows/draft-test', summary: 'Draft test flow', - modules: JSON.stringify(modules) + modules: JSON.stringify(modules), + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await withCompletedTestJob(() => @@ -4434,7 +5510,7 @@ describe('global AI tools', () => { path: 'f/flows/deployed-test', summary: 'Deployed test flow', value: { modules }, - schema: {} + schema: FLOW_NAME_SCHEMA } as any) await withCompletedTestJob(() => @@ -4458,15 +5534,17 @@ describe('global AI tools', () => { }) }) - it('test_run_flow uses the live flow editor test hook when the active editor matches the path', async () => { + // The editor is driven by the key its draft is stored under, which reads and edits of the + // path resolve through too: a staged rename leaves that key where it was. + it('test_run_flow drives the live flow editor by its storage path', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_storage', { path: 'u/admin/live_flow', summary: 'Live flow', value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4477,7 +5555,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_storage', effectivePath: 'u/admin/live_flow' }) const testActiveFlow = vi.fn(async () => 'job-live-flow') @@ -4489,26 +5567,35 @@ describe('global AI tools', () => { path: 'u/admin/live_flow', args: { name: 'Ada' } }, - toolCallbacks, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) }, { testActiveFlow } ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + // What the form submitted, not what the model proposed: the editor runs the flow, but + // the arguments are the user's. The third argument is the chat-mode memory id, + // which only `test_run_flow`'s own `memory_id` supplies. + expect(testActiveFlow).toHaveBeenCalledWith( + 'u/admin/live_flow_storage', + { name: 'Grace' }, + undefined + ) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).not.toHaveBeenCalled() expect(result).toContain('Result (SUCCESS)') }) - it('test_run_flow falls back to preview when the live flow editor test hook returns undefined', async () => { + // A chat flow only shows its memory across turns, so the model has to be able to name + // the conversation it is continuing rather than getting a fresh one every call. + it('test_run_flow passes the memory id it was given to the live editor hook', async () => { seedBackendDraft( 'flow', '', { - path: 'u/admin/live_flow_fallback', - summary: 'Live flow fallback', - value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, - schema: {}, + path: 'u/admin/live_chat_flow', + summary: 'Live chat flow', + value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, + schema: { type: 'object', properties: { user_message: { type: 'string' } } }, edited_by: '', edited_at: '', archived: false, @@ -4520,6 +5607,87 @@ describe('global AI tools', () => { workspace: WORKSPACE, itemKind: 'flow', storagePath: '', + effectivePath: 'u/admin/live_chat_flow' + }) + const testActiveFlow = vi.fn(async () => 'job-live-chat') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { + path: 'u/admin/live_chat_flow', + args: { user_message: 'hi' }, + memory_id: '550e8400-e29b-41d4-a716-446655440000' + }, + toolCallbacks, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).toHaveBeenCalledWith( + '', + { user_message: 'hi' }, + '550e8400-e29b-41d4-a716-446655440000' + ) + }) + + it('test_run_flow gives a chat-enabled flow a conversation when none is named', async () => { + const value = { + modules: [{ id: 'chat_step', value: { type: 'identity' } }], + chat_input_enabled: true + } + seedBackendDraft( + 'flow', + 'u/admin/chat_preview', + { + path: 'u/admin/chat_preview', + summary: 'Chat preview', + value, + schema: { type: 'object', properties: { user_message: { type: 'string' } } }, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + + await withCompletedTestJob(() => + callGlobalTool('test_run_flow', { + path: 'u/admin/chat_preview', + args: { user_message: 'hi' } + }) + ) + + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + memoryId: expect.stringMatching( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ + ), + requestBody: { path: 'u/admin/chat_preview', value, args: { user_message: 'hi' } } + }) + }) + + it('test_run_flow falls back to preview when the live flow editor test hook returns undefined', async () => { + seedBackendDraft( + 'flow', + 'u/admin/live_flow_fallback', + { + path: 'u/admin/live_flow_fallback', + summary: 'Live flow fallback', + value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/live_flow_fallback', effectivePath: 'u/admin/live_flow_fallback' }) const testActiveFlow = vi.fn(async () => undefined) @@ -4536,7 +5704,11 @@ describe('global AI tools', () => { ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + expect(testActiveFlow).toHaveBeenCalledWith( + 'u/admin/live_flow_fallback', + { name: 'Ada' }, + undefined + ) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).toHaveBeenCalledWith({ workspace: WORKSPACE, @@ -4548,8 +5720,223 @@ describe('global AI tools', () => { }) }) + // A flow reaches its run form the way a script does: opened on the flow's own input schema, + // with the dynamic-option picker the schema carries, and the run takes what came back. + it('test_run_flow opens the form on the flow schema and runs what it submitted', async () => { + const modules = [{ id: 'start', value: { type: 'identity' } }] + await callGlobalTool('write_flow', { + path: 'f/flows/formed-flow', + summary: 'Formed flow', + modules: JSON.stringify(modules), + schema: JSON.stringify({ + ...FLOW_NAME_SCHEMA, + 'x-windmill-dyn-select-code': 'export function names() { return ["Ada"] }', + 'x-windmill-dyn-select-lang': 'bun' + }) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'f/flows/formed-flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.args).toEqual({ name: 'Ada' }) + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + // The flow's own dynselect script, which the schema carries rather than a step. + expect({ code: form.code, lang: form.lang }).toEqual({ + code: 'export function names() { return ["Ada"] }', + lang: 'bun' + }) + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/formed-flow', + value: { modules }, + args: { name: 'Grace' } + } + }) + }) + + // The form waits as long as the user does, so which editor is on screen is only known when + // they press Run — checking it when the card appeared would drive an editor they have since + // moved away from. + it('test_run_flow re-checks the editor on screen when the form is submitted', async () => { + const modules = [{ id: 'moved_step', value: { type: 'identity' } }] + seedBackendDraft( + 'flow', + 'u/admin/moved_flow', + { + path: 'u/admin/moved_flow', + summary: 'Moved flow', + value: { modules }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/moved_flow', + effectivePath: 'u/admin/moved_flow' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/moved_flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + // The preview panel moves to another flow while the form sits open. + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/other_flow', + effectivePath: 'u/admin/other_flow' + }) + return form.args + } + }, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { path: 'u/admin/moved_flow', value: { modules }, args: { name: 'Ada' } } + }) + }) + + // The flow may be open in a session tab that isn't the one on screen: driving its editor + // would paint the run into a tab the user is not looking at. + it('test_run_flow previews rather than driving an editor the user is not looking at', async () => { + seedBackendDraft( + 'flow', + 'u/admin/background_flow', + { + path: 'u/admin/background_flow', + summary: 'Background flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/flow_on_screen', + effectivePath: 'u/admin/flow_on_screen' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/background_flow', args: { name: 'Ada' } }, + toolCallbacks, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'u/admin/background_flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + args: { name: 'Ada' } + } + }) + }) + + // What separates a deployed run from a test run of the same flow: the form is built from the + // deployed schema rather than the draft's, and the editor open on that path is left alone — + // it holds the draft, so a deployed run painted into its graph would show steps that are not + // the ones running. + it('run_flow forms on the deployed flow and leaves the live editor alone', async () => { + seedBackendDraft( + 'flow', + 'u/admin/deployed_and_drafted', + { + path: 'u/admin/deployed_and_drafted', + summary: 'Draft of the deployed flow', + value: { modules: [{ id: 'draft_step', value: { type: 'identity' } }] }, + schema: { type: 'object', properties: { draft_only: { type: 'string' } } }, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/deployed_and_drafted', + effectivePath: 'u/admin/deployed_and_drafted' + }) + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'u/admin/deployed_and_drafted', + summary: 'Deployed flow', + value: { modules: [{ id: 'deployed_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA + } as any) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'run_flow', + { path: 'u/admin/deployed_and_drafted', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + }, + { testActiveFlow } + ) + ) + + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(JobService.runFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'u/admin/deployed_and_drafted', + requestBody: { name: 'Grace' }, + skipPreprocessor: true + }) + }) + it('test_run_step previews rawscript steps from the draft flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + // The form offers the fields the step's own code declares, so the step needs a schema + // for `name` to survive it. + stubInferredProperties({ name: { type: 'string' } }) await callGlobalTool('write_flow', { path: 'f/flows/rawscript-step', summary: 'Flow with rawscript', @@ -4607,6 +5994,9 @@ describe('global AI tools', () => { ]) }) + // A script draft carries no schema, so the form infers from the draft content — the + // version about to run. + stubInferredProperties({ name: { type: 'string' } }) await withCompletedTestJob(() => callGlobalTool('test_run_step', { path: 'f/flows/script-step', @@ -4632,7 +6022,10 @@ describe('global AI tools', () => { await callGlobalTool('write_flow', { path: 'f/flows/nested-draft', summary: 'Nested draft flow', - modules: JSON.stringify(nestedModules) + modules: JSON.stringify(nestedModules), + // A subflow step's form is the subflow's own inputs, so `name` needs declaring here + // for it to survive the form. + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await callGlobalTool('write_flow', { path: 'f/flows/parent-flow', @@ -4668,6 +6061,419 @@ describe('global AI tools', () => { }) }) + it('test_run_step runs a deployed subflow step past its preprocessor', async () => { + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'f/flows/deployed-sub', + summary: 'Deployed subflow', + value: { modules: [{ id: 'sub_start', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA + } as any) + await callGlobalTool('write_flow', { + path: 'f/flows/parent-of-deployed', + summary: 'Parent flow', + modules: JSON.stringify([ + { + id: 'call_deployed', + value: { type: 'flow', path: 'f/flows/deployed-sub', input_transforms: {} } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_step', { + path: 'f/flows/parent-of-deployed', + stepId: 'call_deployed', + args: { name: 'Ada' } + }) + ) + + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(JobService.runFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/flows/deployed-sub', + requestBody: { name: 'Ada' }, + skipPreprocessor: true + }) + }) + + // A step is fed by its input transforms, so its arguments are its own and the flow's + // schema describes a different set entirely. Opening the form on the flow's would offer + // fields this job ignores and drop the ones it takes. + it('test_run_step opens the form on the step, not on the flow', async () => { + const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/step-form', + summary: 'Step form flow', + // The flow takes `customer`; the step takes `name`. Nothing links the two. + schema: JSON.stringify({ type: 'object', properties: { customer: { type: 'string' } } }), + modules: JSON.stringify([ + { + id: 'format_name', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + } + ]) + }) + + stubInferredProperties({ name: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { + path: 'f/flows/step-form', + stepId: 'format_name', + args: { name: 'Ada', customer: 'acme' } + }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.schema.properties).toEqual({ name: { type: 'string' } }) + expect(form.runnableKind).toBe('script') + expect(form.summary).toBe('step "format_name"') + // `customer` is the flow's argument, so the step's form never offered it. + expect(form.args).toEqual({ name: 'Ada' }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { content, language: 'bun', args: { name: 'Grace' } } + }) + }) + + // The step runs the draft script's content, so a form built from the deployed schema + // would offer the arguments of code that is not the code about to run. + it('test_run_step opens a script step on the draft schema, not the deployed one', async () => { + const content = 'export async function main(name: string) {\n\treturn `draft ${name}`\n}' + seedBackendDraft('script', 'f/scripts/drifted', { + path: 'f/scripts/drifted', + summary: 'Drifted', + content, + language: 'bun' + }) + await callGlobalTool('write_flow', { + path: 'f/flows/drifted-step', + summary: 'Drifted step flow', + modules: JSON.stringify([ + { + id: 'call_script', + value: { type: 'script', path: 'f/scripts/drifted', input_transforms: {} } + } + ]) + }) + + // Inferred from the draft's content. Never fetching the deployed script is the point: + // its stored schema describes code this run is not about to execute. + stubInferredProperties({ name: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/drifted-step', stepId: 'call_script', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() + expect(form.schema.properties).toEqual({ name: { type: 'string' } }) + }) + + // No parser emits `password`, so the draft's stored schema is the only thing carrying it. + // Rebuilding the form's fields from the content would offer the secret as a plain text + // box, and the literal typed into it would reach the job's arguments unminted. + it('test_run_step keeps the password marking of a drafted script step', async () => { + seedBackendDraft('script', 'f/scripts/secretful', { + path: 'f/scripts/secretful', + summary: 'Secretful', + content: 'export async function main(token: string) {\n\treturn 1\n}', + language: 'bun', + schema: { + type: 'object', + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + }) + await callGlobalTool('write_flow', { + path: 'f/flows/secretful-step', + summary: 'Secretful step flow', + modules: JSON.stringify([ + { + id: 'call_secretful', + value: { type: 'script', path: 'f/scripts/secretful', input_transforms: {} } + } + ]) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/secretful-step', stepId: 'call_secretful', args: {} }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + expect(form.schema.properties).toMatchObject({ token: { password: true } }) + }) + + // The entrypoint override is declared by no schema, so it has to be added after the form + // rather than proposed into it — anything that conforms arguments to a schema drops it, + // and the preprocessor then silently runs its `main`. + it('test_run_step keeps the preprocessor entrypoint out of the form and on the job', async () => { + const content = 'export async function preprocessor(event: string) {\n\treturn event\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/preprocessed', + summary: 'Preprocessed flow', + modules: JSON.stringify([{ id: 'start', value: { type: 'identity' } }]), + preprocessor_module: JSON.stringify({ + id: 'preprocessor', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + }) + }) + + vi.mocked(inferArgs).mockClear() + stubInferredProperties({ event: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/preprocessed', stepId: 'preprocessor', args: { event: 'signup' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + // Inferred against the preprocessor entrypoint, not `main`. + expect(vi.mocked(inferArgs).mock.calls[0][3]).toBe('preprocessor') + expect(form.schema.properties).toEqual({ event: { type: 'string' } }) + expect(form.args).toEqual({ event: 'signup' }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + content, + language: 'bun', + args: { _ENTRYPOINT_OVERRIDE: 'preprocessor', event: 'signup' } + } + }) + }) + + // The form IS the consent, so a dismissed one must leave the script unrun. + it('run_script starts no job when the user cancels the form', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/deployed', + summary: 'Deployed', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/deployed', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async () => undefined } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('The user cancelled the run form') + expect(result).toContain('Do not call run_script again') + }) + + // job_args.test.ts owns what each step of the argument pipeline does; this owns that + // run_script still runs them. Delete a call from runThroughForm and every one of those + // unit tests still passes, so one call has to cross all of them here. + it('run_script puts the proposed arguments through the whole pipeline', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/everything', + schema: { + properties: { + count: { type: 'number' }, + ratio: { type: 'number' }, + size: { type: 'number' }, + token: { type: 'string', password: true }, + locked: { type: 'string', default: 'fixed', disabled: true }, + doc: { type: 'string', contentEncoding: 'base64' } + } + } + } as any) + + const bytes = 'QUJD'.repeat(1024) + const statuses: any[] = [] + let shown: Record | undefined + let cleared: string[] | undefined + let reset: string[] | undefined + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { + path: 'f/scripts/everything', + args: { + count: '7', + ratio: 'abc', + size: '$var:u/admin/batch_size', + token: 'hunter2', + locked: 'tampered', + doc: bytes, + force_delete: true + } + }, + { + ...toolCallbacks, + setToolStatus: (_toolId: string, status: any) => statuses.push(status), + requestRunArgs: async (_toolId, form) => { + shown = form.args + cleared = form.clearedKeys + reset = form.resetKeys + // What the user does with the form: attaches the file no model can produce, + // and names a variable for the secret it was not allowed to fill. + return { ...form.args, doc: bytes, token: '$var:u/ada/prod_api_key' } + } + } + ) + ) + + // Coerced, cleared, left alone, reset, dropped and emptied of bytes — every rule reached + // through the tool rather than called directly. The proposed secret is not emptied: it is + // already in the model's own tool call in the same stored record, and PasswordArgInput + // mints whatever the field opens with before the job sees it. + expect(shown).toEqual({ + count: 7, + size: '$var:u/admin/batch_size', + token: 'hunter2', + locked: 'fixed' + }) + expect(cleared).toEqual(['ratio']) + expect(reset).toEqual(['locked']) + + // The bytes belong in the job request and nowhere else: the card is persisted, and a + // file small enough to survive truncation would otherwise reach the model whole. + expect(JobService.runScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/everything', + requestBody: { + count: 7, + size: '$var:u/admin/batch_size', + locked: 'fixed', + doc: bytes, + token: '$var:u/ada/prod_api_key' + }, + skipPreprocessor: true + }) + expect(result).toContain('does not declare force_delete') + expect(result).toContain('') + // The bytes are the value; the reference is not, and the run page shows it for this + // same job. + expect(result).not.toContain(bytes) + expect(JSON.stringify(statuses)).not.toContain(bytes) + expect(result).toContain('$var:u/ada/prod_api_key') + expect(JSON.stringify(statuses)).toContain('$var:u/ada/prod_api_key') + // Named, or an emptied field reads as the user having deleted the value and the next + // call proposes the same bytes again. + for (const named of ['ratio', 'doc', 'locked']) expect(result).toContain(named) + }) + + // The form is its own confirmation, so it never reaches processToolCall's second gate. + // Plan mode can be switched on while it sits open, and the job must not start. + it('run_script starts no job when plan mode is entered while the form is open', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/noargs', + schema: { properties: {} } + } as any) + + let planning = false + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/noargs', args: {} }, + { + ...toolCallbacks, + isPlanModeActive: () => planning, + requestRunArgs: async (_toolId, form) => { + planning = true + return form.args + } + } + ) + + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('plan mode is active') + }) + + // A form that reaches the screen mints a proposed secret on mount, which the gate after + // the user answers is too late to unmake — so plan mode arriving during the fetch counts. + it('run_script opens no form when plan mode is entered during the schema fetch', async () => { + let planning = false + vi.mocked(ScriptService.getScriptByPath).mockImplementationOnce(async () => { + planning = true + return { + path: 'f/scripts/pw', + schema: { properties: { token: { type: 'string', password: true } } } + } as any + }) + + let formOpened = false + const result = await callGlobalTool( + 'run_script', + { path: 'f/scripts/pw', args: { token: 'hunter2' } }, + { + ...toolCallbacks, + isPlanModeActive: () => planning, + requestRunArgs: async (_toolId, form) => { + formOpened = true + return form.args + } + } + ) + + expect(formOpened).toBe(false) + expect(JobService.runScriptByPath).not.toHaveBeenCalled() + expect(result).toContain('plan mode is active') + }) + + it('run_script runs the arguments the user submitted, not the ones proposed', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/greet', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/greet', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) } + ) + ) + + expect(JobService.runScriptByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/scripts/greet', + requestBody: { name: 'Grace' }, + skipPreprocessor: true + }) + // The model must not assume its proposal is what ran. + expect(result).toContain('Ran with arguments: {"name":"Grace"}') + }) + + // The arguments are already in the call this result answers, and every way the form's + // own differ from the proposed ones has its own clause — so an untouched form has + // nothing to name, and naming it anyway pays for the copy on every later iteration. + it('run_script names the arguments only when the user changed them', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/greet', + schema: { properties: { name: { type: 'string' } } } + } as any) + + const result = await withCompletedTestJob(() => + callGlobalTool( + 'run_script', + { path: 'f/scripts/greet', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, form) => form.args } + ) + ) + + expect(result).not.toContain('Ran with arguments') + expect(result).toContain('unedited') + }) + it('test_run_step lists nested step ids when a step is not found', async () => { await callGlobalTool('write_flow', { path: 'f/flows/nested-step-error', @@ -5236,7 +7042,7 @@ describe('prepareGlobalSystemMessage', () => { it('dispatches to the registered handler with the session id and default limit of 20', async () => { const callbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } const handler = vi.fn(() => ({ - aiResult: 'runs output. Next step: call get_job_logs.', + aiResult: 'runs output. Next step: call get_run.', uiMessage: 'Listed 1 app run', toolResult: '[{"job_id":"job-1","component":"backend.1","status":"completed","created_at":1718000000000,"started_at":1718000000000,"duration_ms":1000}]' @@ -5245,7 +7051,7 @@ describe('prepareGlobalSystemMessage', () => { const result = await callGlobalTool('list_app_runs', {}, callbacks, { sessionId: 'sess-runs' }) - expect(result).toBe('runs output. Next step: call get_job_logs.') + expect(result).toBe('runs output. Next step: call get_run.') expect(handler).toHaveBeenCalledWith({ sessionId: 'sess-runs', limit: 20 }) expect(callbacks.setToolStatus).toHaveBeenLastCalledWith('test-list_app_runs', { content: 'Listed 1 app run', @@ -5358,6 +7164,8 @@ describe('session-only preview tools gating', () => { expect(names).not.toContain('list_app_runs') expect(names).not.toContain('search_dom') expect(names).not.toContain('read_dom') + // Not withheld: without it the side panel has no route to a deployed run at all. + expect(names).toContain('run_script') // other tools are still present expect(names).toContain('write_script') }) @@ -5370,6 +7178,7 @@ describe('session-only preview tools gating', () => { expect(names).toContain('list_app_runs') expect(names).toContain('search_dom') expect(names).toContain('read_dom') + expect(names).toContain('run_script') // The session set is the full globalTools minus capability-gated tools: // this environment is not Chromium, so take_screenshot is withheld (DOM // capture is only faithful on Blink). search_dom / read_dom are not gated. @@ -5471,6 +7280,21 @@ describe('session-only preview tools gating', () => { }) }) +describe('read_skill', () => { + // Every path is enabled by default now, so the listing is what keeps the tool to + // skills: without it the model could name any resource holding a string `content` + // and have it read back. + it('refuses a path that is not a skill in the workspace, without reading it', async () => { + localStorage.clear() + userStore.set({ username: 'bob', email: 'bob@windmill.dev', workspace_id: WORKSPACE } as any) + + const res = await callGlobalTool('read_skill', { path: 'u/someone/private-notes' }) + + expect(res).toContain('not one of the skills available') + expect(vi.mocked(ResourceService.getResourceValue)).not.toHaveBeenCalled() + }) +}) + describe('update_user_instructions', () => { function makeHelpers(initial = '') { let value = initial diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index c34f8f17b6..ec41effdf4 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1,3 +1,4 @@ +import { randomUUID } from '$lib/utils/uuid' import { AppService, AzureTriggerService, @@ -18,9 +19,11 @@ import { SqsTriggerService, VariableService, WebsocketTriggerService, + WorkerService, WorkspaceService } from '$lib/gen' import { createTwoFilesPatch } from 'diff' +import { deepEqual } from 'fast-equals' import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' import { $ScriptLang } from '$lib/gen/schemas.gen' import type { @@ -30,7 +33,6 @@ import type { Flow, FlowModule, FlowValue, - Job, ListableApp, ListableResource, ListableVariable, @@ -48,6 +50,16 @@ import { STARTER_RUNNABLE_KEY, type FrameworkKey } from '$lib/components/raw_apps/templates' +import { + coerceArgsToSchema, + dropUndeclaredArgs, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs, + stripFileArgs +} from '$lib/components/job_args' +import { processSecretArgs } from '$lib/components/secretArgUtils' +import { PLAN_MODE_MESSAGES } from '../planModeMessages' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' import type { RawAppDomQuery } from '$lib/components/raw_apps/rawAppDom' @@ -83,6 +95,16 @@ import { } from '../flow/inlineScriptsUtils' import { searchNpmPackagesTool } from '../script/core' import type { McpServer } from './mcpTools' +import { logFeatureUsage } from '$lib/utils/featureUsage' +import { isSkillEnabled } from '../skills/enabledSkills' +import { + listSkillResources, + readSkillBody, + skillNameFromPath, + truncateChars, + truncateForPrompt +} from '../skills/skillResources' +import { MAX_SKILL_DESCRIPTION_LENGTH, MAX_SKILL_INSTRUCTIONS_LENGTH } from '../skills/skillMd' import { getDatatableSdkReference, getFlowPrompt, @@ -105,12 +127,14 @@ import { createToolDef, droppedOptionKeys, createSearchHubScriptsTool, - executeFlowStepTestRun, executeTestRun, findAndReplace, isHubPath, + resolveFlowStepRun, + SPECIAL_MODULE_IDS, type CreatedResourceTriggerKind, type PreviewCardKind, + type RunFormDisplay, type Tool, type ToolCallbacks, type ToolDisplayAction @@ -127,6 +151,7 @@ import type { SessionArtifactsStore } from '../artifacts/artifactsState.svelte' import type { Runnable } from '$lib/components/apps/inputType' import { UserDraft } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' +import type { Schema } from '$lib/common' import { inferArgs } from '$lib/infer' import { resourceRequestSchema, @@ -157,6 +182,7 @@ import { workspaceStore } from '$lib/stores' import { getWorkspaceRole, type RoleLookup } from '$lib/user' +import { refreshSuperadmin } from '$lib/refreshUser' import { get } from 'svelte/store' import { canonicalDraftSideValue, @@ -164,7 +190,7 @@ import { getDraftDiffValues } from '$lib/utils_draft_deploy' import { changedLineIndices, draftDeployedPatch, windowPatch } from './draftDiff' -import { getFlowRunDetails } from './flowRunTree' +import { getRun, summarizeRun } from './flowRunTree' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' import type { UserDraftItemKind } from '$lib/gen' @@ -228,7 +254,6 @@ const VARIABLE_MASKED_NOTE = 'Note: variable values are never shown in chat — the diff marks whether the value changed without revealing it.\n\n' const SECRET_UNCOMPARABLE_NOTE = 'Note: this is a SECRET variable — its value is never shown and cannot be compared, so it may ALSO have changed beyond what this diff shows.\n\n' -import { apiCatalogTools } from './apiCatalogTools' const ITEM_TYPES = [ 'script', @@ -267,6 +292,8 @@ const ACTIVE_GLOBAL_EDITOR_DRAFTS: readonly { export type GlobalActiveEditorContext = { type: ActiveGlobalEditorType path: string + /** The key the draft is stored under, which `path` leaves behind on a rename. */ + storagePath: string isLiveDraft: true } @@ -281,8 +308,8 @@ export type GlobalActivePreviewContext = { * location: a tab can host a legacy app whose hash is app state, and a filter value * can be free text the user typed. Build it with `previewLocationContext`. */ location: string - /** The row whose drawer is open on that page. The list pages drop the anchor when - * their drawer closes, so its absence means no row is open. */ + /** The item open in its editor: a list page row whose drawer is open, or the item a + * session tab edits. Its absence means no item is open. */ open?: string } @@ -636,7 +663,7 @@ const writeVariableSchema = variableRequestSchema.extend({ .string() .optional() .describe( - 'The value of the variable. Omit it to leave the value alone — required only when creating a new variable, or when changing a secret variable into a non-secret one. Never invent or guess the value of an existing variable: you cannot read it, and a "$var:..." reference is NOT a valid value (that syntax only references a variable from inside a resource). Omitting it keeps whatever the draft already holds, so a value you set earlier in this conversation stays set; discard_local_draft abandons it.' + 'The value of the variable. Omit it to leave the value alone — required only when creating a new variable, or when changing a secret variable into a non-secret one. Never invent or guess the value of an existing variable: you cannot read it, and a "$var:..." reference is NOT a valid value (a variable cannot reference itself). Omitting it keeps whatever the draft already holds, so a value you set earlier in this conversation stays set; discard_local_draft abandons it.' ), is_secret: z .boolean() @@ -679,17 +706,13 @@ const searchResourceTypesSchema = z.object({ .describe('Max number of resource types to return. Defaults to 5.') }) -const getJobLogsSchema = z.object({ - id: z.string().describe('The UUID of the job to fetch logs for.') -}) - -const getFlowRunDetailsSchema = z.object({ - id: z.string().describe('The UUID of the flow run to inspect.'), +const getRunSchema = z.object({ + id: z.string().describe('The UUID of the run (job) to inspect.'), step: z .string() .optional() .describe( - 'Step to drill into for its result (returned in full up to 12k chars), addressed by the step ids shown in the tree: "b" for a top-level step, "b/c" for a step inside a subflow, "b[12]" for iteration 12 of a loop or attempt 12 of a retried step (1-based), composable as "b[12]/c". Omit to get the whole per-step tree.' + 'Step to drill into for its result (returned in full up to 12k chars), addressed by the step ids shown in the tree: "b" for a top-level step, "b/c" for a step inside a subflow, "b[12]" for iteration 12 of a loop or attempt 12 of a retried step (1-based), composable as "b[12]/c". Omit to get the run itself.' ) }) @@ -715,6 +738,17 @@ const listRunsSchema = z.object({ .describe('Max number of runs to return, most recent first. Defaults to 30.') }) +// `GET /workers/list` hides workers from a caller without the devops role by +// returning an empty list, not an error, when HIDE_WORKERS_FOR_NON_ADMINS is set. +// The flag is not visible here, so absence is only provable for a devops or +// superadmin caller; every other caller gets the hedge even where nothing is hidden. +const NO_WORKERS_VISIBLE_MESSAGE = + 'No workers came back. This does NOT establish that no workers are running: an instance can hide workers from callers without the devops role, and it does so by returning an empty list rather than an error. ' + + 'Tell the user you cannot see any workers and that worker visibility may be restricted for your account, and suggest they check the Workers page themselves. Never state that no workers are online or that the instance has none.' +const NO_WORKERS_CONNECTED_MESSAGE = + 'No workers are connected to this instance (none pinged in the last 5 minutes). Queued runs will stay queued until a worker starts.' +const WORKER_PAGE_SIZE = 100 + const deleteWorkspaceItemSchema = z.object({ type: itemTypeSchema, path: z.string().describe('Workspace path of the item to delete.'), @@ -856,7 +890,9 @@ const testRunArgsSchema = z .record(z.string(), z.any()) .nullable() .optional() - .describe('Arguments to pass to the runnable. Omit or pass null when no arguments are needed.') + .describe( + 'Arguments to pass to the runnable. Omit or pass null when no arguments are needed. An argument typed as a resource (format "resource-" in the input schema) takes the bare string "$res:" as its whole value — never an object wrapper like {"$res": ""}, and never a plain path, both of which reach the runnable unresolved. Same for a variable, with "$var:". The prefixed string can also sit in a nested field, e.g. {"gh_auth": {"token": "$var:g/all/gh_token"}}.' + ) const backgroundArgSchema = z .boolean() @@ -889,13 +925,38 @@ const testRunScriptSchema = z.object({ const testRunScriptToolDef = createToolDef( testRunScriptSchema, 'test_run_script', - 'Execute a preview-style test run of a script by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a script by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', + { strict: false } +) + +const runScriptSchema = z.object({ + path: z.string().describe('Workspace path of the deployed script to run.'), + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema +}) + +const runScriptToolDef = createToolDef( + runScriptSchema, + 'run_script', + 'Run a DEPLOYED script for real, under the user\'s own permissions. Fill in every argument you can infer: the user gets an argument form prefilled with `args` and decides what runs. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call. A required file is the user\'s to attach, so call this even when you cannot supply one rather than asking in chat. Use only when the user names the deployed version ("the deployed X", "in production", "for real"); otherwise use test_run_script.', { strict: false } ) const testRunFlowSchema = z.object({ path: z.string().describe('Workspace path of the flow to test.'), args: testRunArgsSchema, + // A refinement rather than z.guid(): that emits `format`/`pattern` into the tool schema, + // which some providers' function-schema subsets reject. + memory_id: z + .string() + .refine((value) => z.guid().safeParse(value).success, { + message: 'memory_id must be a UUID' + }) + .optional() + .describe( + 'Chat-mode flows only. A UUID naming the conversation this turn belongs to, whose memory the agent steps read: reuse the same one across calls to test memory and follow-ups, and omit it for a one-off turn in a conversation of its own. Generate the UUID yourself so you can pass it again.' + ), background: backgroundArgSchema, wait_seconds: waitSecondsArgSchema }) @@ -903,7 +964,21 @@ const testRunFlowSchema = z.object({ const testRunFlowToolDef = createToolDef( testRunFlowSchema, 'test_run_flow', - 'Execute a preview-style test run of a flow by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a flow by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', + { strict: false } +) + +const runFlowSchema = z.object({ + path: z.string().describe('Workspace path of the deployed flow to run.'), + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema +}) + +const runFlowToolDef = createToolDef( + runFlowSchema, + 'run_flow', + 'Run a DEPLOYED flow for real, under the user\'s own permissions. Fill in every argument you can infer: the user gets an argument form prefilled with `args` and decides what runs. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call. A required file is the user\'s to attach, so call this even when you cannot supply one rather than asking in chat. Use only when the user names the deployed version ("the deployed X", "in production", "for real"); otherwise use test_run_flow.', { strict: false } ) @@ -918,7 +993,7 @@ const testRunStepSchema = z.object({ const testRunStepToolDef = createToolDef( testRunStepSchema, 'test_run_step', - 'Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists.', + "Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists. `args` are the step's OWN inputs, not the flow's: a step is normally fed by its input transforms, so send what that step's code takes, not what the flow takes. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.", { strict: false } ) @@ -1268,7 +1343,7 @@ const buildGlobalSystemPrompt = ( // right now: the system prompt is the cached prefix, so a line appearing and // disappearing between turns costs more cache than the tool call it saves. const activePreviewRule = previewTools - ? '\n- If the user message includes an ACTIVE PREVIEW section, that is the page the side panel is showing — resolve "this page", "here" and "it" against it, and against `open` (the row the page is anchored at, whose drawer the user opened) when there is one. It already tells you what get_preview_status would, so do not call that tool to learn what is on screen; call it only to check the panel\'s *other* tabs.' + ? '\n- If the user message includes an ACTIVE PREVIEW section, that is the page the side panel is showing — resolve "this page", "here" and "it" against it, and against `open` (the item the user has open in its editor) when there is one. It already tells you what get_preview_status would, so do not call that tool to learn what is on screen; call it only to check the panel\'s *other* tabs.' : '' const pipelineBullet = `- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow.${pipelineAlphaNote}` // Hosting and edition come from the hostname and a store the app populates at init, so @@ -1312,17 +1387,16 @@ Rules: ${pipelineBullet} - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Do the same for a raw app: run test_run_app_runnable on each backend runnable you wrote or changed before saying the app works. A bundle that compiles proves nothing about whether the runnables run. An inline runnable executes the app's draft code; a path runnable executes the DEPLOYED script/flow it names, so a path runnable aimed at something you have not deployed fails here — that failure is the point: report it and offer to deploy that one target. The app itself does not need deploying to be tested. -- Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run. -- To see what a flow run actually did per step — statuses and results across the whole execution tree, subflow steps and loop iterations included — use get_flow_run_details with the run id (it also works while the flow is still running). Pass step to read one step's result in full (capped at 12k chars). Prefer it over get_job_logs when you need step results rather than logs. +- Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_run with a returned id to see what that run was called with, what it returned and what it logged — without starting a new test run. +- get_run also covers what a flow run did per step — statuses and results across the whole execution tree, subflow steps and loop iterations included — and works while the flow is still running. Pass step to read one step's result in full (capped at 12k chars). - Use open_page to show a workspace page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, or Workspace settings on a specific tab (e.g. "open the failed runs of f/foo/bar", "open the schedule for X", "open the git sync settings"). Carry over every filter the user described — Runs takes the page's whole filter set (time window, path, user, folder, label, tag, worker, trigger kind, args/result, ...), so don't drop a criterion just because it wasn't in the request's main clause. Only the pages listed for this user in the tool are available; don't offer pages that aren't listed. Don't use it as a substitute for list_runs when you just need the data yourself. -- Whenever you ask the user to perform a manual step in the UI — fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting — call open_page in the same message, targeted at that item (pass open with its path to land in its edit drawer, or the page's filters otherwise). Never just describe where to click. +- Whenever you ask the user to perform a manual step in the UI — fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting — call open_page in the same message, targeted at that item (pass open with its path to land in its editor, or the page's filters otherwise). Never just describe where to click. - When the user is happy with the changes and wants to review or deploy them, use open_page with page "compare" — it opens the Compare & Deploy review page.${ previewTools ? ' By default it preselects the items this chat modified; pass items (":" entries) to control the selection' : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. -- For a Windmill operation no other tool covers (workers, queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. -- runScriptByPath / runFlowByPath from the API catalog run the DEPLOYED version of an item. Use them only when the user explicitly asks to run the deployed version, and read the item with read_workspace_item version: "deployed" first so the arguments match the deployed input schema (a draft may have different inputs). To test something you are editing or just wrote, always use test_run_script, test_run_flow, or test_run_step — they run the draft. +- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script or run_flow only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For those two, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema. test_run_script, test_run_flow, test_run_step, run_script and run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. test_run_step's form is the step's own inputs, not the flow's. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -1332,7 +1406,7 @@ ${pipelineBullet} - Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node — this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node — use ONLY those to add or change pipeline nodes, never write_script for a pipeline node — they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). - To inspect what actually rendered in a running raw app (verify an edit landed on screen, diagnose a blank/empty or wrong view, answer "what's showing"), use search_dom (regex over the live HTML) and read_dom (a line-numbered window). Pass a \`selector\` to scope to an element — prefer the selector from a DOM element chip the user attached — or omit it for the whole page. When a chip lists an \`app_path\`, pass it too so the RIGHT app is read (several previews can be open; a query without \`app_path\` hits the visible one). The DOM is read live and is never in context; no match means the element isn't rendered. Both need the raw app preview open. -- get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected. +- get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_run with it. Use this when a backend call errors or returns something unexpected. ${ isChromiumBrowser() ? `- When the user raises how a raw app looks (something is off, or they want the design or layout improved), call take_screenshot to see what they are looking at before changing anything. Reach for it when the request is about appearance, not to review your own edits, which you can read back from the code. It needs the raw app preview open (open_preview kind="raw_app").` @@ -1357,7 +1431,9 @@ Flows: - Use patch_flow_json for structural flow edits and write_flow for full flow rewrites. Raw apps: +- The app tools below only work on raw (code) apps. \`rawApp\` says which: false is a drag-and-drop app: you can list it and read its metadata, but not read its contents, edit it or deploy it. Check it before offering to change an app. - read_workspace_item returns app metadata only. Use read_app_file for file and inline runnable contents. +- A draft app is reachable by nobody; deploying is what exposes its backend runnables. deploy_workspace_item says so when the deploy widens who may open the app: anonymous means anyone with the URL, without logging in; guest means anyone the instance's identity provider authenticates, member of this workspace or not. Relay that in plain words and carry on. This is disclosure, not a gate: do not stop and ask for permission, and do not refuse the deploy. You cannot change who may open an app from chat; it is set on the app's deploy settings. - Use write_app_file, patch_app_file, and delete_app_file for frontend files. - Use write_app_runnable and delete_app_runnable for backend runnables. - Use init_app only after confirming framework, path, and summary with the user. @@ -1367,15 +1443,20 @@ Data Tables: - Datatables are workspace-scoped managed PostgreSQL databases, shared across the workspace (not owned by any single app). They must be configured by the user in their workspace settings (Workspace settings → Data Tables); they cannot be created via SQL. - Use list_datatables to discover the available datatables and their tables. Reuse an existing table rather than creating a duplicate. If list_datatables reports none, this is a blocking prerequisite — tell the user to set up a datatable in their workspace settings and stop; do not assume a "main" datatable exists or call exec_datatable_sql. - Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries. -- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step. +- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.${ + isCloudHosted() + ? '' + : ` +- A raw app may use a datatable through a role (\`data.roles\` in its raw_app.yaml). When working on such an app, pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can.` + } - When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.${ skills.length > 0 ? ` Skills: -- Skills are reusable instruction sets curated for this workspace, each covering a specific kind of task. The available skills are listed below by name and description. -- When a user's request matches a skill's description, call read_skill with its exact name to load the full instructions BEFORE acting, then follow them. -${skills.map((s) => `- ${s.name}: ${s.description}`).join('\n')}` +- Skills are reusable instruction sets available in this workspace, each covering a specific kind of task. The available skills are listed below by resource path and description. +- When a user's request matches a skill's description, call read_skill with its exact path to load the full instructions BEFORE acting, then follow them. +${skills.map((s) => `- ${s.path}: ${s.description}`).join('\n')}` : '' }${ mcpServers.length > 0 @@ -1462,6 +1543,8 @@ function serializeWorkspaceItemForRead(item: WorkspaceItem): unknown { path: item.path, summary: item.summary, value: summarizeAppValue(item.value as AppDraftValue), + rawApp: item.rawApp, + executionMode: item.executionMode, isDraft: item.isDraft } } @@ -1511,36 +1594,6 @@ function variableToItem(variable: ListableVariable): WorkspaceItem { } } -// Compact metadata for one run. The raw Job carries args/result/logs/raw_code -// which can be huge — list_runs returns only what's needed to identify a run. -function summarizeRun(job: Job): Record { - const base = { - id: job.id, - job_kind: job.job_kind, - path: job.script_path, - created_by: job.created_by, - created_at: job.created_at, - started_at: job.started_at, - schedule_path: job.schedule_path, - is_flow_step: job.is_flow_step, - tag: job.tag, - worker: job.worker - } - if ('success' in job) { - // CompletedJob - return { - ...base, - status: job.canceled ? 'canceled' : job.success ? 'success' : 'failure', - duration_ms: job.duration_ms - } - } - // QueuedJob (running or still waiting in the queue) - return { - ...base, - status: job.running ? 'running' : 'queued' - } -} - // ============= App helpers ============= type BackendRunnableInput = z.infer @@ -1718,6 +1771,9 @@ function appToItem(app: ListableApp | AppWithLastVersion, includeValue: boolean) path: app.path, summary: app.summary, value: includeValue ? ((app as AppWithLastVersion).value as AppDraftValue) : undefined, + // The server omits this flag rather than sending false, so its absence in the + // response is a known false — not a value this listing failed to fetch. + rawApp: app.raw_app ?? false, isDraft: false } } @@ -1785,13 +1841,28 @@ function getInlineRunnableContent( return { content: runnable.inlineScript?.content ?? '', runnable } } +// appSourceToDraftValue drops a low-code app's `grid`, so converting one here would stage a +// code-app draft at its path: a false picture of the app that every later read answers from, +// and a deploy the server then refuses. +async function getRawAppByPath(workspace: string, path: string): Promise { + const app = await AppService.getAppByPath({ workspace, path }) + // Only an explicit false: the draft-only branch of get_app carries no version, and so + // no `raw_app`, and must not read as low-code. + if (app.raw_app === false) { + throw new Error( + `"${path}" is a low-code app. This chat only edits code-based apps — open it in the app editor instead.` + ) + } + return app +} + async function loadAppValueForRead(path: string, workspace: string): Promise { const draft = await getGlobalDraft(workspace, 'app', path) if (draft && draft.value && typeof draft.value === 'object' && 'files' in draft.value) { return draft.value as AppDraftValue } - const app = await AppService.getAppByPath({ workspace, path }) + const app = await getRawAppByPath(workspace, path) return appSourceToDraftValue(app, app) } @@ -1801,7 +1872,7 @@ async function loadAppDraftValue(path: string, workspace: string): Promise = { } } +/** Whether the server would admit a guest here: the deployment has to support guests at + * all, and the instance and workspace switches both have to be on. `guest` is stored on + * an app even when none of that holds, so the mode alone never settles who can open it. + * `undefined` when a switch read fails — neither proven live nor proven inert. */ +async function guestAccessIsLive(workspace: string): Promise { + const [usage, settings] = await Promise.all([ + WorkspaceService.getGuestUsage({ workspace }).catch(() => undefined), + WorkspaceService.getPublicSettings({ workspace }).catch(() => undefined) + ]) + if (usage === undefined || settings === undefined) { + return undefined + } + return !!(usage.available && usage.instance_enabled && settings.guest_access_enabled) +} + +/** Who may open an app, from the mode stored on it. An app sits in `guest` mode whether + * or not anyone is admitted by it, so reporting the mode bare would say strangers can + * open an app that admits members only. Say it is inert rather than hide it, as the + * app's deploy settings do. */ +async function describeAppExposure( + workspace: string, + mode: string | undefined +): Promise { + if (mode !== 'guest' || (await guestAccessIsLive(workspace)) !== false) { + return mode + } + return 'guest, but inert: the instance or workspace admits no guest, so this app admits members only' +} + async function readWorkspaceItem( type: WorkspaceItemType, path: string, @@ -1997,6 +2097,19 @@ async function readWorkspaceItem( case 'app': { // Returns lightweight metadata only — file/runnable contents come via read_app_file. const app = await AppService.getAppByPath({ workspace, path }) + // A grid is not files and runnables: summarizing one reports an empty app. Name the + // kind instead. + const executionMode = await describeAppExposure(workspace, app.policy?.execution_mode) + if (app.raw_app === false) { + return { + type: 'app', + path: app.path, + summary: app.summary, + rawApp: false, + executionMode, + isDraft: false + } + } const value = appSourceToDraftValue(app) const metadata = summarizeAppValue(value) return { @@ -2004,6 +2117,8 @@ async function readWorkspaceItem( path: app.path, summary: value.summary, value: metadata as unknown as AppDraftValue, + rawApp: app.raw_app, + executionMode, isDraft: false } } @@ -2216,7 +2331,7 @@ function getResourceInstructions(): string { - Reading a variable returns \`{ type: 'variable', path, summary?, isSecret, isDraft }\` — never its value, secret or not. \`isSecret\` tells you whether the value is encrypted. - \`write_variable\` takes \`{ path, value?, is_secret?, description?, account?, is_oauth?, expires_at?, value_expires_at?, labels? }\`. Creating a variable needs \`value\` and \`is_secret\`; editing one needs only the fields you are changing. Omitting \`value\` keeps the stored value, which is the only way to edit a secret variable — you cannot read its value, so passing any \`value\` you did not get from the user destroys it. - For secret fields in a resource value, do NOT inline the raw secret. Create a Variable first with \`is_secret: true\`, then in the resource value reference it as \`"$var:path/to/variable"\`. -- Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. These are references FROM a resource value; never store a \`$var:\` string as a variable's own value. +- Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. The same strings are also how a resource or variable is passed as a run argument (see the run-argument rule in the resource reference below); what they are never valid as is a variable's own value. - When deploying drafts that depend on each other (e.g., a resource and the variables it references), deploy the variables first. - Use \`search_resource_types\` to discover valid \`resource_type\` names and their JSON Schemas. Match the resource value to that schema. - For OAuth resources, the \`is_oauth: true\` flag is managed by Windmill's OAuth flow; global mode generally creates manual resources, not OAuth ones. @@ -2264,7 +2379,9 @@ function getInstructions( } } -export type AiSkillListItem = { name: string; description: string } +/** A skill the user turned on, as the prompt and the `/` picker see it. `path` + * is the `ai_skill` resource and the model-facing id; `name` is its basename. */ +export type AiSkillListItem = { path: string; name: string; description: string } /** Live session facts appended to the GLOBAL system prompt for session chats. * Provided by the session runtime as a resolver (copilot must not import the @@ -2327,15 +2444,37 @@ export function getSessionContextPromptSection(ctx: SessionPromptContext): strin return lines.join('\n') } -/** `/` picker entry: a workspace skill or a built-in session action. The kind - * drives the picker's category grouping; entries without one are ungrouped. */ -export type ChatCommandItem = AiSkillListItem & { kind?: 'action' | 'skill' } +/** `/` picker entry: a selected skill or a built-in session action. The kind + * drives the picker's category grouping; entries without one are ungrouped. + * Only skills carry a `path` — built-in actions run locally and have no resource. */ +export type ChatCommandItem = { + name: string + description: string + path?: string + kind?: 'action' | 'skill' +} -/** Fetch the workspace's AI skills (name + description) for the global system prompt. */ +/** + * The skills in play in this workspace, for the global system prompt: every + * readable `ai_skill` resource except the ones this user turned off. + */ export async function loadWorkspaceSkills(workspace: string): Promise { if (!workspace) return [] try { - return await WorkspaceService.listAiSkills({ workspace }) + // A truncated listing still carries most of the workspace, and the drawer is + // where that is surfaced; dropping everything here would silently empty the + // Skills section instead. + return (await listSkillResources(workspace)).skills + .filter((s) => isSkillEnabled(workspace, s.path)) + .map(({ path, name, description }) => ({ + path, + name, + // Every description goes into the system prompt on every turn, and any + // resource of this type can be selected — including ones written through + // git sync or the resource editor, which never saw the authoring form's + // bounds. One unbounded description would crowd out the conversation. + description: truncateChars(description, MAX_SKILL_DESCRIPTION_LENGTH) + })) } catch (e) { console.error('Failed to load AI skills', e) return [] @@ -2343,32 +2482,56 @@ export async function loadWorkspaceSkills(workspace: string): Promise = { def: createToolDef( readSkillSchema, 'read_skill', - 'Load the full instructions for a workspace AI skill by name. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' + 'Load the full instructions for an AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' ), planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readSkillSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${parsed.name}"...` }) + const name = skillNameFromPath(parsed.path) + toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${name}"...` }) try { - const skill = await WorkspaceService.getAiSkill({ workspace, name: parsed.name }) - toolCallbacks.setToolStatus(toolId, { content: `Read skill "${parsed.name}"` }) - return `Skill: ${skill.name}\nDescription: ${skill.description}\n\nInstructions:\n${skill.instructions}` + // The prompt lists the skills in play, but the tool takes a path the model + // composed, so what may be read is checked here rather than assumed. Against + // the listing, not just the off-switch: any other path is now enabled too, + // and without this the tool reads any resource holding a string `content` — + // the user's own access, but not what "load a skill" says it does. + const isSkill = (await listSkillResources(workspace)).skills.some( + (s) => s.path === parsed.path + ) + if (!isSkill || !isSkillEnabled(workspace, parsed.path)) { + toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not available` }) + return `"${parsed.path}" is not one of the skills available to this chat. Only the paths listed under "Skills" in the system prompt can be read.` + } + // Bounded here rather than in the reader: any `ai_skill` resource is in play, + // including ones written through git sync or the resource editor + // that never passed the authoring form's limits, and an unbounded body + // would exhaust the context on one tool call. The editor reads the same + // resource untruncated, so opening a long skill cannot rewrite it short. + const instructions = truncateForPrompt( + await readSkillBody(workspace, parsed.path), + MAX_SKILL_INSTRUCTIONS_LENGTH + ) + toolCallbacks.setToolStatus(toolId, { content: `Read skill "${name}"` }) + // Whether a skill is actually reached for. No key: the path is + // workspace-authored text. + logFeatureUsage('ai_session', 'skill_read', { workspace }) + return `Skill: ${parsed.path}\n\nInstructions:\n${instructions}` } catch (e) { const msg = e instanceof Error ? e.message : String(e) toolCallbacks.setToolStatus(toolId, { - content: `Error reading skill "${parsed.name}"`, + content: `Error reading skill "${name}"`, error: msg }) - return `Failed to read skill "${parsed.name}": ${msg}. Check the name against the Skills list in the system prompt.` + return `Failed to read skill "${parsed.path}": ${msg}. Check the path against the Skills list in the system prompt.` } } } @@ -2702,7 +2865,7 @@ const openPageFullSchema = z.object({ .string() .optional() .describe( - 'Schedules/Triggers/Variables/Resources: exact item path to open in the edit drawer, e.g. f/foo/my_schedule. Use it whenever the user should act on one specific item (e.g. fill in credentials) so they land directly in its editor.' + 'Schedules/Triggers/Variables/Resources: exact item path to open in its editor, e.g. f/foo/my_schedule — in a session, a preview tab of its own instead of the list page. Use it whenever the user should act on one specific item (e.g. fill in credentials) so they land directly in its editor.' ), summary: z .string() @@ -2831,7 +2994,7 @@ function buildOpenPageDefSchema( } const OPEN_PAGE_DESCRIPTION = - 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy review page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its edit drawer opens directly). Use page "compare" when the user wants to review and deploy pending changes (the items field controls which changes are preselected). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' + 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy review page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its editor opens directly). Use page "compare" when the user wants to review and deploy pending changes (the items field controls which changes are preselected). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' // Non-arg inputs the URL builder needs: the chat's operating workspace (the compare // page cannot fall back to its own store default inside a session preview) and the @@ -3326,15 +3489,23 @@ export const globalTools: Tool<{}>[] = [ // (it filters before the cap; query filters after). if ((parsed.page ?? 1) === 1) { const draftCountByType = new Map() + const prefix = parsed.path_prefix for (const draft of await listGlobalDrafts(workspace)) { if (!types.includes(draft.type)) continue - if (parsed.path_prefix && !draft.path.startsWith(parsed.path_prefix)) continue + // A draft's staged name is often not where it is stored: the editor parks + // a new script, flow or app at a generated `draft_` path, and a + // rename stages the new name over the old path. The server drops + // draft-only rows under any narrowing filter, leaving this pass their + // only source, so either name has to satisfy the prefix. + if (prefix && !draft.path.startsWith(prefix) && !draft.draftPath?.startsWith(prefix)) + continue const count = draftCountByType.get(draft.type) ?? 0 if (count >= limit) continue draftCountByType.set(draft.type, count + 1) byKey.set(getWorkspaceItemKey(draft.type, draft.path, draft.triggerKind), { ...draft, - value: undefined + value: undefined, + schema: undefined }) } } @@ -3372,7 +3543,17 @@ export const globalTools: Tool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: `Read draft ${parsed.type} "${parsed.path}"` }) - return JSON.stringify(serializeWorkspaceItemForRead(draft), null, 2) + // The mode reported is the draft's, which is what deploying it will write. The + // deployed app's own mode is a different question, asked with version: + // "deployed", and must not be fetched here. + const executionMode = + parsed.type === 'app' + ? await describeAppExposure( + workspace, + (draft.value as AppDraftValue)?.policy?.execution_mode + ) + : undefined + return JSON.stringify(serializeWorkspaceItemForRead({ ...draft, executionMode }), null, 2) } toolCallbacks.setToolStatus(toolId, { @@ -3589,33 +3770,68 @@ export const globalTools: Tool<{}>[] = [ const parsed = testRunScriptSchema.parse(ctx.args) return testRunScriptByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the script')}`, + // No requiresConfirmation: the argument form is the confirmation, and the bypass posture + // answers it with what the form opened with — a decision made for the user, so the + // posture's own list has to name it. One thing does run before Run: see the note on + // the form's SchemaForm. + bypassedByAutoAccept: true, + confirmationMessage: 'Run a test of a script', + streamingLabel: 'Preparing the test form...', queuedLabel: (args) => `Test ${args?.path ?? 'the script'}`, showDetails: true, autoCollapseDetails: false }, + { + def: runScriptToolDef, + fn: async (ctx) => { + const parsed = runScriptSchema.parse(ctx.args) + return runDeployedScript(parsed, ctx) + }, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + confirmationMessage: 'Run a deployed script', + streamingLabel: 'Preparing the run form...', + queuedLabel: (args) => `Run ${args?.path ?? 'a script'}`, + showDetails: true, + autoCollapseDetails: false + }, { def: testRunFlowToolDef, fn: async (ctx) => { const parsed = testRunFlowSchema.parse(ctx.args) return testRunFlowByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow', queuedLabel: (args) => `Test ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false }, + { + def: runFlowToolDef, + fn: async (ctx) => { + const parsed = runFlowSchema.parse(ctx.args) + return runDeployedFlow(parsed, ctx) + }, + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the run form...', + confirmationMessage: 'Run a deployed flow', + queuedLabel: (args) => `Run ${args?.path ?? 'a flow'}`, + showDetails: true, + autoCollapseDetails: false + }, { def: testRunStepToolDef, fn: async (ctx) => { const parsed = testRunStepSchema.parse(ctx.args) return testRunFlowStepByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => - `Run a test of step "${args?.stepId ?? ''}" in ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow step', queuedLabel: (args) => `Test step "${args?.stepId ?? ''}" of ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false @@ -3624,7 +3840,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( listRunsSchema, 'list_runs', - "List recent runs (jobs), most recent first. Optionally filter by path, creator, label, or status. Returns compact metadata only — use get_job_logs with a returned id to read a run's logs." + 'List recent runs (jobs), most recent first. Optionally filter by path, creator, label, or status. Returns compact metadata only — use get_run with a returned id to see what a run was called with, returned and logged.' ), planModeSafe: true, showDetails: true, @@ -3651,24 +3867,42 @@ export const globalTools: Tool<{}>[] = [ }, { def: createToolDef( - getFlowRunDetailsSchema, - 'get_flow_run_details', - "Inspect a flow run's execution tree: per-step statuses and truncated results, including subflow steps, loop iterations, branches, and retries. Works on running flows too. Pass step to fetch one step's result in full (up to 12k chars)." + z.object({}), + 'list_workers', + 'List the workers connected to this Windmill instance (those that pinged in the last 5 minutes), with their worker group, custom tags, seconds since their last ping, and jobs executed. Pair with list_runs to diagnose a stuck queue: runs queued on a tag no listed worker picks up will never start. Three blind spots to report rather than reason past: an empty result states whether no worker is connected or whether workers may be hidden from you, so relay the one it gives instead of picking; a missing custom_tags can mean tags are hidden from you, not unset; and only the 100 most recently pinging workers are listed, so on a bigger instance a tag none of them carries may still be served.' ), planModeSafe: true, showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = getFlowRunDetailsSchema.parse(args) + fn: async ({ toolId, toolCallbacks }) => { + toolCallbacks.setToolStatus(toolId, { content: 'Listing workers...' }) + const pings = await WorkerService.listWorkers({ perPage: WORKER_PAGE_SIZE }) + if (pings.length === 0) { + // Both role stores resolve asynchronously, and an unloaded one must not read as + // an absent role — that hedges the answer this branch exists to give plainly. + // No-ops once they hold a value. + await refreshSuperadmin() + const hiddenFromCaller = !get(superadmin) && !get(devopsRole) + const message = hiddenFromCaller ? NO_WORKERS_VISIBLE_MESSAGE : NO_WORKERS_CONNECTED_MESSAGE + toolCallbacks.setToolStatus(toolId, { + content: hiddenFromCaller ? 'No workers visible' : 'No workers connected', + result: message + }) + return message + } + const workers = pings.map((w) => ({ + worker: w.worker, + worker_group: w.worker_group, + custom_tags: w.custom_tags, + last_ping: w.last_ping, + jobs_executed: w.jobs_executed + })) + const note = + workers.length === WORKER_PAGE_SIZE + ? `Only the first ${WORKER_PAGE_SIZE} workers are listed; more may be connected.` + : undefined + const result = JSON.stringify({ workers, ...(note ? { note } : {}) }, null, 2) toolCallbacks.setToolStatus(toolId, { - content: parsed.step - ? `Fetching result of step ${parsed.step} in run ${parsed.id}...` - : `Inspecting flow run ${parsed.id}...` - }) - const result = await getFlowRunDetails(workspace, parsed.id, parsed.step) - toolCallbacks.setToolStatus(toolId, { - content: parsed.step - ? `Fetched result of step ${parsed.step} in run ${parsed.id}` - : `Inspected flow run ${parsed.id}`, + content: `Listed ${workers.length} worker(s)`, result }) return result @@ -3676,34 +3910,31 @@ export const globalTools: Tool<{}>[] = [ }, { def: createToolDef( - getJobLogsSchema, - 'get_job_logs', - 'Fetch the logs of a job by its id. Use this to inspect the output of an existing run.' + getRunSchema, + 'get_run', + "Inspect one run: its status, arguments, result and logs, plus — for a flow — the per-step execution tree with each step's status and truncated result (subflow steps, loop iterations, branches and retries included). Works on running jobs too. Pass step to fetch one step's result in full (up to 12k chars)." ), planModeSafe: true, showDetails: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = getJobLogsSchema.parse(args) + const parsed = getRunSchema.parse(args) toolCallbacks.setToolStatus(toolId, { - content: `Fetching logs for job ${parsed.id}...` + content: parsed.step + ? `Fetching result of step ${parsed.step} in run ${parsed.id}...` + : `Inspecting run ${parsed.id}...` }) - const logs = await JobService.getJobLogs({ - workspace, - id: parsed.id, - // Always suppress the "to remove ansi colors, use: sed ..." hint the - // backend otherwise prepends — it is noise for the model and is not - // actual ANSI stripping (the raw logs are returned either way). - removeAnsiWarnings: true - }) - const hasLogs = typeof logs === 'string' && logs.trim().length > 0 - const result = hasLogs ? logs : 'No logs available for this job.' + const { text, jobId } = await getRun(workspace, parsed.id, parsed.step) toolCallbacks.setToolStatus(toolId, { - content: hasLogs - ? `Fetched logs for job ${parsed.id}` - : `No logs available for job ${parsed.id}`, - result + content: parsed.step + ? `Fetched result of step ${parsed.step} in run ${parsed.id}` + : `Inspected run ${parsed.id}`, + result: text, + // The card reads the run itself from here; the model only ever gets `text`. + ...(jobId + ? { inspectedRun: { jobId, workspace, runId: parsed.id, step: parsed.step } } + : {}) }) - return result + return text } }, { @@ -4202,13 +4433,10 @@ export const globalTools: Tool<{}>[] = [ }, // Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy) ...getDatatableTools(), - // Workspace DuckLake readiness (storage prerequisite check for pipelines) + // Workspace DuckLake: pipeline storage prerequisite, and declared measures ...getDucklakeTools(), // Read-only tools over files the user attached to the conversation - ...fileTools, - // Search + call access to the backend API endpoint catalog, for operations - // no dedicated tool covers - ...apiCatalogTools + ...fileTools ] // Tools that only make sense inside an AI session (they drive the session's @@ -4232,8 +4460,8 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ /** * The global tool set for a given chat: the full `globalTools` for a session - * chat, or `globalTools` minus the session-only preview tools for the regular - * global side-panel chat. + * chat, or `globalTools` minus the preview tools for the regular global + * side-panel chat. */ export function globalToolsFor({ sessionPreview }: { sessionPreview: boolean }): Tool<{}>[] { const tools = sessionPreview @@ -4274,7 +4502,13 @@ type WriteDraftCtx = { export type SessionToolHelpers = { sessionId?: string } export type GlobalToolHelpers = SessionToolHelpers & { - testActiveFlow?: (args?: Record) => Promise + /** Runs the flow editor mounted on `storagePath`, if one is. `memoryId` names the + * chat-mode conversation the turn belongs to. */ + testActiveFlow?: ( + storagePath: string, + args?: Record, + memoryId?: string + ) => Promise attachedFiles?: AttachedFilesStore // Read/write the user-level Global instructions. `setUserInstructions` persists the // value and rebuilds the system message so the change applies on the next chat-loop @@ -4303,15 +4537,23 @@ function operatingWorkspaceFromHelpers(helpers: unknown): string | undefined { return (helpers as GlobalToolHelpers | undefined)?.operatingWorkspace } -function activeFlowTestFromCtx( +// Drive a live editor only for the flow on screen: several can be open at once (session tabs), +// and a run painted into a background tab is a side effect the user never sees. Undefined +// sends the caller to a preview run, which reports into the chat alone. The hook is bound to +// that editor's storage path — the same key reads and edits of `path` resolve through, so a +// staged rename cannot send the run to a different editor than the one being edited. +function liveFlowTestHookFromCtx( ctx: { workspace: string; helpers?: unknown }, path: string -): GlobalToolHelpers['testActiveFlow'] | undefined { +): ((args?: Record, memoryId?: string) => Promise) | undefined { const activeEditor = getActiveGlobalEditorContext(ctx.workspace) if (activeEditor?.type !== 'flow' || activeEditor.path !== path) { return undefined } - return (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + const testActiveFlow = (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + return ( + testActiveFlow && ((args, memoryId) => testActiveFlow(activeEditor.storagePath, args, memoryId)) + ) } export type OpenPreviewHandler = (req: { @@ -4916,11 +5158,14 @@ const SCRIPT_SPEC: WriteSpec = { language: args.language, kind: 'script' } - // Infer the arg schema from the content at save time, like the editor does, - // so the persisted draft is the single source of truth at deploy. Keep the - // previous schema (or empty) on failure rather than blanking it. + // Into the schema the base carries, as the editor does at save: `inferArgs` re-seeds + // each arg from the properties it is handed, and those are the only copy of + // `password`, enums, formats and titles — no parser emits them. A clone, so a parse + // failure leaves the previous schema rather than half of one. try { - const schema = emptySchema() + const schema = structuredClone( + draft.schema?.properties ? draft.schema : emptySchema() + ) as Schema await inferArgs(draft.language, draft.content, schema) draft.schema = schema } catch (e) { @@ -5071,7 +5316,7 @@ function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promis // is always the model echoing the reference syntax back instead of a real value. if (args.value === `$var:${args.path}`) { throw new Error( - `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. The "$var:" syntax only references a variable from inside a resource value. Omit value to keep the current one.` + `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. Omit value to keep the current one.` ) } return writeDraft(VARIABLE_SPEC, 'variable', args.path, args, ctx, { override: args.override }) @@ -5080,16 +5325,56 @@ function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promis async function loadScriptForEdit( path: string, workspace: string -): Promise<{ content: string; language: ScriptLang; summary?: string }> { +): Promise<{ + content: string + language: ScriptLang + summary?: string + schema?: Record +}> { const draft = await getGlobalDraft(workspace, 'script', path) if (draft) { if (typeof draft.value !== 'string' || !draft.language) { throw new Error(`Draft script "${path}" is missing content or language.`) } - return { content: draft.value, language: draft.language, summary: draft.summary } + return { + content: draft.value, + language: draft.language, + summary: draft.summary, + schema: draft.schema as Record | undefined + } } const script = await ScriptService.getScriptByPath({ workspace, path }) - return { content: script.content, language: script.language, summary: script.summary } + return { + content: script.content, + language: script.language, + summary: script.summary, + schema: script.schema as Record | undefined + } +} + +/** The fields a test form offers, for code that may never have been deployed. The stored + * schema wins wherever it declares fields — a draft's or the deployed script's; one that + * declares nothing, or that speaks for an entrypoint other than the one about to run, is + * inferred here from the content instead. */ +async function schemaForTestRun(script: { + content: string + language: ScriptLang + schema?: Record + /** A preprocessor takes the arguments of its own entrypoint, not of `main`. */ + entrypoint?: 'preprocessor' +}): Promise> { + // Emptily declared is not declared: a stored `properties: {}` means the schema predates + // the arguments the code now takes, so infer rather than offer a form with no fields. + // A stored schema speaks for one entrypoint, so it can never answer for an override. + if (!script.entrypoint && Object.keys(script.schema?.properties ?? {}).length > 0) + return script.schema! + const schema = emptySchema() + try { + await inferArgs(script.language, script.content, schema, script.entrypoint) + } catch (e) { + console.error('Failed to infer script schema for the test run form', e) + } + return schema as unknown as Record } async function editScript( @@ -5115,18 +5400,21 @@ async function editScript( async function loadFlowDraftValue( path: string, workspace: string -): Promise<{ flow: FlowDraftValue; summary?: string }> { + // `isDraft` says which of the two this came from. Reported here because the draft lookup + // is a request of its own: a caller that needs to know would otherwise repeat it. +): Promise<{ flow: FlowDraftValue; summary?: string; isDraft: boolean }> { const draft = await getGlobalDraft(workspace, 'flow', path) if (draft) { if (draft.value === undefined || typeof draft.value === 'string') { throw new Error(`Draft flow "${path}" has no value.`) } - return { flow: draft.value as FlowDraftValue, summary: draft.summary } + return { flow: draft.value as FlowDraftValue, summary: draft.summary, isDraft: true } } const flow = await FlowService.getFlowByPath({ workspace, path }) return { flow: { value: flow.value, schema: flow.schema, groups: flow.value.groups ?? null }, - summary: flow.summary + summary: flow.summary, + isDraft: false } } @@ -5294,33 +5582,55 @@ function flowDraftValueForPreview(flowDraft: FlowDraftValue): FlowValue { return flowDraftAsEditableInput(flowDraft).value } +/** + * The conversation a test run of a chat-enabled flow belongs to. The server refuses such a + * run without one, and it is a query parameter rather than a flow argument, so there is no + * way for the caller to supply it through `args`. A fresh id each time is the right default: + * a test run is its own conversation, not a turn appended to one someone is reading. + */ +export function chatMemoryId(value: FlowValue): string | undefined { + return value.chat_input_enabled ? randomUUID() : undefined +} + async function loadScriptForFlowStep( moduleValue: { path: string; hash?: string }, workspace: string -): Promise<{ content: string; language: ScriptLang }> { +): Promise<{ content: string; language: ScriptLang; schema?: Record }> { const draft = await getGlobalDraft(workspace, 'script', moduleValue.path) if (draft) { if (typeof draft.value !== 'string' || !draft.language) { throw new Error(`Draft script "${moduleValue.path}" is missing content or language.`) } - return { content: draft.value, language: draft.language } + return { + content: draft.value, + language: draft.language, + // The draft's own: no parser emits `password`, so a schema rebuilt from the content + // would offer a secret argument as a plain field and take the literal into the job. + schema: draft.schema as Record | undefined + } } const script = moduleValue.hash ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) - return { content: script.content, language: script.language } + return { + content: script.content, + language: script.language, + schema: script.schema as Record | undefined + } } -async function loadDraftFlowPreviewValue( +async function loadSubflowForFlowStep( path: string, workspace: string -): Promise { - if (!(await getGlobalDraft(workspace, 'flow', path))) { - return undefined - } +): Promise<{ previewValue?: FlowValue; schema?: Record }> { const nestedFlow = await loadFlowDraftValue(path, workspace) - return flowDraftValueForPreview(nestedFlow.flow) + return { + // Only a draft is previewed; a deployed subflow is run by path, as its parent flow + // would run it. The schema describes whichever of the two that leaves. + previewValue: nestedFlow.isDraft ? flowDraftValueForPreview(nestedFlow.flow) : undefined, + schema: nestedFlow.flow.schema ?? undefined + } } // Leaf of a workspace path (last segment), for human-readable confirmation @@ -5334,90 +5644,437 @@ async function testRunScriptByPath( args: z.infer, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx + const { workspace } = ctx const script = await loadScriptForEdit(args.path, workspace) - const testArgs = normalizeTestRunArgs(args.args) + const schema = await schemaForTestRun(script) - return executeTestRun({ - jobStarter: () => - JobService.runScriptPreview({ - workspace, - requestBody: { - path: args.path, - content: script.content, - args: testArgs, - language: script.language - } - }), + return runThroughForm( + { + path: args.path, + schema, + summary: script.summary, + kind: 'test', + code: script.content, + lang: script.language, + // Never "deployed" here: the code about to run is the draft the model is still + // writing, and a line telling it to re-read the deployed schema would send it + // to the wrong version. + schemaNoun: 'script', + toolName: 'test_run_script', + proposed: args.args, + startMessage: `Running test for script "${args.path}"...`, + contextName: 'script', + // Its own loop: the model is told to test and iterate, so the posture answers the + // form with what it opened with rather than parking the loop on a card. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runScriptPreview({ + workspace, + requestBody: { + path: args.path, + content: script.content, + args: submitted, + language: script.language + } + }) + }, + ctx + ) +} + +/** The "do not call again" half is load-bearing: without it the model re-proposes the + * call, which re-opens the form the user just dismissed, and Stop becomes their only + * way out. */ +const runFormCancelled = (toolName: string, noun: string) => + `The user cancelled the run form. The ${noun} did NOT run. Do not call ${toolName} again unless the user asks for it.` + +/** The model only needs to see what the user changed, and nothing bounds an object or + * array argument the form let them paste into. */ +const MAX_SUBMITTED_ARGS_LENGTH = 4000 + +/** The card's own copy is bounded separately, and far higher: it is what the details pane + * renders, and JobArgs stops rendering the JSON in full at this size regardless. */ +const MAX_PERSISTED_ARGS_LENGTH = 100_000 + +/** One run through an argument form: conform what the model proposed to the schema of the + * version about to run, open the form on it, then run whatever came back. Both tools that + * run a script are this, differing only in where the schema comes from and how the job + * starts — so the user meets one card whichever they asked for. */ +type FormRunSpec = { + path: string + schema: Record + summary?: string + kind: 'run' | 'test' + /** The code a test run is about to preview, so its form can offer the same dynamic-option + * pickers the script editor's test panel does. Omitted for a deployed run, which names a + * path instead. */ + code?: string + lang?: ScriptLang + /** How the lines the model reads back name the version this ran: telling it to re-read + * the "deployed schema" of a draft would send it to the wrong code. */ + schemaNoun: string + toolName: string + proposed: Record | null | undefined + startMessage: string + /** What runs: the jobs-tray kind, and the noun the card's own prose reads. */ + contextName: 'script' | 'flow' + /** What the lines the model reads back call the thing that ran. Defaults to + * `contextName`, which a flow step is not: it runs a script or a subflow but is neither. */ + noun?: string + /** What names the run where its path would not: the jobs-tray row, and the two + * background-job sentences the model reads. Those quote it, so it carries none. */ + label?: string + /** Whether the bypass posture may answer this form with what it opened with. */ + autoAcceptable?: boolean + background?: boolean + detachAfterMs?: number + startJob: (submitted: Record) => Promise +} + +async function runThroughForm(spec: FormRunSpec, ctx: WriteDraftCtx): Promise { + const { workspace, toolId, toolCallbacks } = ctx + // Asked of the posture, not of the tool: every run tool is auto-acceptable, so a + // host with no form would otherwise run one on the model's arguments alone, in any + // posture. What a bypass answers is a decision the user already made; without it there + // is no consent to be had here and nothing to fall back on. + const postureAnswers = Boolean( + spec.autoAcceptable && toolCallbacks.shouldAutoAcceptToolConfirmations?.(spec.toolName) + ) + const noun = spec.noun ?? spec.contextName + if (!toolCallbacks.requestRunArgs && !postureAnswers) { + return `This chat cannot show a run form, so a ${noun} cannot be run from here.` + } + + // processToolCall gates plan mode once, before the schema fetch, and this form is its own + // confirmation so it never reaches that gate again. Repeated wherever a write follows: a + // mounted field mints on its own, which no later gate can unmake. + const blockedByPlanMode = (): string | undefined => { + if (!toolCallbacks.isPlanModeActive?.()) return undefined + toolCallbacks.onToolBlockedByPlanMode?.() + toolCallbacks.setToolStatus(toolId, { + content: PLAN_MODE_MESSAGES.blockedLabel, + isLoading: false, + isStreamingArguments: false, + error: PLAN_MODE_MESSAGES.blockedResult, + blockedByPlanMode: true + }) + return PLAN_MODE_MESSAGES.blockedResult + } + const blockedBeforeForm = blockedByPlanMode() + if (blockedBeforeForm) return blockedBeforeForm + + const schema = spec.schema + // Whether to ask is the only question decided here. What a mounted field would hold — a + // default, a synthesised empty, whether Run lights up — is the form's own business: any + // second derivation of it here can start a run the form itself would refuse. + const autoAccepted = postureAnswers + const strippedKeys: string[] = [] + const coerced = autoAccepted + ? undefined + : coerceArgsToSchema(normalizeTestRunArgs(spec.proposed), schema) + let proposed: Record + let resetKeys: string[] + let undeclaredKeys: string[] + if (coerced) { + resetKeys = coerced.resetKeys + undeclaredKeys = coerced.undeclaredKeys + // Left as the model proposed it, minted by the widget the field mounts: a reference put + // here instead would be normalised away by the nested form an object secret renders as. + proposed = stripFileArgs(coerced.args, schema as any, strippedKeys) + } else { + // Both rules hold against every caller, not only the ones a form stands in front of: + // an undeclared argument has no field anywhere, and a disabled one is nobody's to set. + // Without the rest of the coercion, which answers what a mounted widget would show. + const declared = dropUndeclaredArgs(normalizeTestRunArgs(spec.proposed), schema) + undeclaredKeys = declared.undeclaredKeys + const enforced = enforceDisabledDefaults(declared.args, schema) + resetKeys = enforced.resetKeys + // In the widget's stead: with no form there is no PasswordArgInput to turn a proposed + // secret into a reference, and the job's arguments outlive the run. + try { + proposed = await processSecretArgs(enforced.args, schema as any, workspace) + } catch (e) { + const message = `Failed to store the sensitive arguments of "${spec.path}": ${e}` + toolCallbacks.setToolStatus(toolId, { + content: message, + isLoading: false, + isStreamingArguments: false, + error: message + }) + return message + } + } + const form: RunFormDisplay = { + path: spec.path, + summary: spec.summary || undefined, + kind: spec.kind, + runnableKind: spec.contextName, + schema: autoAccepted ? undefined : schema, + code: autoAccepted ? undefined : spec.code, + lang: autoAccepted ? undefined : spec.lang, + submitted: autoAccepted || undefined, + args: proposed, + clearedKeys: coerced?.clearedKeys.length ? coerced.clearedKeys : undefined, + resetKeys: resetKeys.length ? resetKeys : undefined, + strippedKeys: strippedKeys.length ? strippedKeys : undefined + } + + // Files only: nothing rewrites `runForm.args` after this, so bytes left in it outlive the + // size guard that covers `parameters`. + const persisted = { ...form, args: redactFileArgs(proposed, schema as any) } + + toolCallbacks.setToolStatus(toolId, { + content: autoAccepted + ? spec.startMessage + : `Waiting for you to confirm the arguments of "${spec.path}"`, + runForm: persisted, + // Not the raw tool-call arguments: the card settles on what the form opened with. + // Only settles it — the raw proposal still renders while the call streams in. + parameters: persisted.args, + isLoading: true + }) + + // `form`, not `persisted`: a password field mints from what it opens with. + const submitted = toolCallbacks.requestRunArgs + ? await toolCallbacks.requestRunArgs(toolId, form, { autoAccepted }) + : proposed + if (!submitted) { + toolCallbacks.setToolStatus(toolId, { + content: `Run of "${spec.path}" cancelled by user`, + isLoading: false, + isStreamingArguments: false, + error: 'Cancelled by user', + declinedByUser: true + }) + return runFormCancelled(spec.toolName, noun) + } + + const blockedBeforeRun = blockedByPlanMode() + if (blockedBeforeRun) return blockedBeforeRun + + // Every job leaves through here, so this is where a sensitive argument becomes a reference: + // the form mints as the user types and the bypass mints in its stead, but a host answering + // the form its own way — the eval harness does — would hand over a literal. Idempotent, so + // the two that already minted pay a walk and no round trip. + let toRun: Record + try { + toRun = await processSecretArgs(submitted, schema as any, workspace) + } catch (e) { + const message = `Failed to store the sensitive arguments of "${spec.path}": ${e}` + toolCallbacks.setToolStatus(toolId, { + content: message, + isLoading: false, + isStreamingArguments: false, + error: message + }) + return message + } + + // The card's details pane must show what ran, not what was proposed. Bytes are marked by + // size because the card is persisted; everything else stands as the run page shows it for + // the same job. + const forCard = redactFileArgs(toRun, schema as any) + // The transcript is re-cloned into IndexedDB on every save and a form carries whatever was + // pasted into it, so past what the pane would render the card reads the arguments off the + // job instead. Only once there is a job to read them from: substituting the marker any + // earlier would leave a run that never started showing nothing but the marker. + const oversized = JSON.stringify(forCard).length > MAX_PERSISTED_ARGS_LENGTH + if (!oversized) { + toolCallbacks.setToolStatus(toolId, { parameters: forCard }) + } + + const outcome = await executeTestRun({ + jobStarter: async () => { + const jobId = await spec.startJob(toRun) + // The form's own submitted flag flips a round trip earlier, when the user presses + // Run; only from here is there a job for a stopped turn to say it left running. + toolCallbacks.markRunFormStarted?.(toolId) + if (oversized) { + toolCallbacks.setToolStatus(toolId, { parameters: { reason: 'WINDMILL_TOO_BIG' } }) + } + return jobId + }, workspace, toolCallbacks, toolId, - startMessage: `Running test for script "${args.path}"...`, - contextName: 'script', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path + startMessage: spec.startMessage, + contextName: spec.contextName, + actionNoun: spec.kind === 'test' ? 'test' : 'run', + background: spec.background, + detachAfterMs: spec.detachAfterMs, + label: spec.label ?? spec.path }) + + const schemaNoun = `${spec.schemaNoun} schema` + // Only what the form could make no reading of: a wrong-typed value it can read is + // converted silently, since the field then shows what the run carries and there is + // nothing to report. + const clearedKeys = coerced?.clearedKeys ?? [] + const cleared = clearedKeys.length + ? `\nThe ${schemaNoun} declares ${clearedKeys.join(', ')}, but you sent ${clearedKeys.length > 1 ? 'them in shapes' : 'it in a shape'} with no reading in the declared ${clearedKeys.length > 1 ? 'types' : 'type'}, so the ${clearedKeys.length > 1 ? 'fields opened' : 'field opened'} empty and the run did not carry ${clearedKeys.length > 1 ? 'them' : 'it'}. Re-read the input schema and match ${clearedKeys.length > 1 ? 'their declared types' : 'its declared type'}.` + : '' + const reset = resetKeys.length + ? `\nThe ${schemaNoun} disables ${resetKeys.join(', ')}, so the run used ${resetKeys.length > 1 ? 'their defaults' : 'its default'} rather than the proposed ${resetKeys.length > 1 ? 'values' : 'value'}. Do not propose ${resetKeys.length > 1 ? 'them' : 'it'} again.` + : '' + // Nothing renders these, so the model is the only one who can be told they went nowhere. + const undeclared = undeclaredKeys.length + ? `\nThe ${schemaNoun} does not declare ${undeclaredKeys.join(', ')}, so ${undeclaredKeys.length > 1 ? 'they were' : 'it was'} not sent — no run form in Windmill offers a field the schema does not name. Re-read the input schema and use the arguments it declares.` + : '' + // Otherwise an emptied field reads as the user having deleted it, and the next call + // proposes the same bytes again. + const stripped = strippedKeys.length + ? `\n${strippedKeys.join(', ')} ${strippedKeys.length > 1 ? 'are file arguments' : 'is a file argument'}, so the form opened ${strippedKeys.length > 1 ? 'them' : 'it'} empty for the user to attach. ${strippedKeys.length > 1 ? 'They are' : 'It is'} theirs to provide, not yours: do not propose ${strippedKeys.length > 1 ? 'them' : 'it'} again.` + : '' + // Redacted for the model alone: what it proposed is already in its own tool call, but a + // secret the user typed into the form would be entering its context here. + const redacted = redactFileArgs(redactSecretArgs(toRun, schema as any), schema as any) + const submittedJson = JSON.stringify(redacted) + const shown = + submittedJson.length > MAX_SUBMITTED_ARGS_LENGTH + ? submittedJson.slice(0, MAX_SUBMITTED_ARGS_LENGTH) + '... (truncated)' + : submittedJson + // Naming them costs a copy of arguments already in the call above, and cleared/reset/ + // stripped name every way the form's own differ from the proposed ones — so only what + // the user changed is news. + const ran = deepEqual(redacted, proposed) + ? 'Ran with the arguments the form opened with, unedited.' + : `Ran with arguments: ${shown}` + return `${ran}${cleared}${reset}${stripped}${undeclared}\n${outcome}` +} + +async function runDeployedScript( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace } = ctx + // No getDraft: this runs the script as it is live, so the form has to offer the + // inputs the live version accepts and not a draft's. + const script = await ScriptService.getScriptByPath({ workspace, path: args.path }) + return runThroughForm( + { + path: args.path, + schema: (script.schema as Record) ?? {}, + summary: script.summary, + kind: 'run', + schemaNoun: 'deployed', + toolName: 'run_script', + proposed: args.args, + startMessage: `Running "${args.path}"...`, + contextName: 'script', + // Bypassable like a test run: the posture is the user's standing answer, and a form + // it parks on is a card nobody is watching. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runScriptByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the script's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the script + // its own output instead. + skipPreprocessor: true + }) + }, + ctx + ) +} + +async function runDeployedFlow( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace } = ctx + // No live editor is driven here as a test run drives one: that editor holds the draft, and a + // deployed run painted into its graph would show steps that are not the ones running. + const flow = await FlowService.getFlowByPath({ workspace, path: args.path }) + return runThroughForm( + { + path: args.path, + schema: (flow.schema as Record) ?? {}, + summary: flow.summary, + kind: 'run', + // No code/lang: the dynamic-option pickers come from the deployed flow, not an inline copy. + schemaNoun: 'deployed', + toolName: 'run_flow', + proposed: args.args, + startMessage: `Running "${args.path}"...`, + contextName: 'flow', + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runFlowByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the flow's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the flow + // its own output instead. + skipPreprocessor: true + }) + }, + ctx + ) } async function testRunFlowByPath( args: z.infer, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const testArgs = normalizeTestRunArgs(args.args) - const testActiveFlow = activeFlowTestFromCtx(ctx, args.path) + const { workspace } = ctx + // The schema must be in hand before the form is built, and the value rides along from the + // same read so the fields and the previewed flow are one version. With an editor open on + // this path this reads its in-memory cell rather than the network. + const flow = await loadFlowDraftValue(args.path, workspace) + const schema = (flow.flow.schema as Record | null | undefined) ?? {} - if (testActiveFlow) { - return executeTestRun({ - jobStarter: async () => { - const jobId = await testActiveFlow(testArgs) + return runThroughForm( + { + path: args.path, + schema, + summary: flow.summary, + kind: 'test', + // A flow's dynamic-option pickers are one script stored on the schema itself, which is + // where the editor's own test form reads them from (FlowPreviewContent). + code: schema['x-windmill-dyn-select-code'], + lang: schema['x-windmill-dyn-select-lang'], + // Never "deployed": a test run previews the draft, so a line sending the model to the + // deployed schema would name the wrong version. + schemaNoun: 'flow', + toolName: 'test_run_flow', + proposed: args.args, + startMessage: `Starting flow test run for "${args.path}"...`, + contextName: 'flow', + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: async (submitted) => { + // An open editor runs its own in-memory flow and paints the run in its graph. + // Resolved here rather than before the form: the form waits as long as the user + // does, and the editor on screen when they press Run is the one it belongs in. + const jobId = await liveFlowTestHookFromCtx(ctx, args.path)?.(submitted, args.memory_id) if (jobId) { return jobId } - - const flow = await loadFlowDraftValue(args.path, workspace) + const value = flowDraftValueForPreview(flow.flow) return JobService.runFlowPreview({ workspace, + memoryId: args.memory_id ?? chatMemoryId(value), requestBody: { path: args.path, - value: flowDraftValueForPreview(flow.flow), - args: testArgs + value, + args: submitted } }) - }, - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) - } - - const flow = await loadFlowDraftValue(args.path, workspace) - - return executeTestRun({ - jobStarter: () => - JobService.runFlowPreview({ - workspace, - requestBody: { - path: args.path, - value: flowDraftValueForPreview(flow.flow), - args: testArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) + } + }, + ctx + ) } async function testRunFlowStepByPath( @@ -5426,21 +6083,61 @@ async function testRunFlowStepByPath( ): Promise { const { workspace, toolId, toolCallbacks } = ctx const flow = await loadFlowDraftValue(args.path, workspace) - const flowValue = flowDraftValueForPreview(flow.flow) - const testArgs = normalizeTestRunArgs(args.args) - - return executeFlowStepTestRun({ - flowValue, + const resolved = await resolveFlowStepRun({ + flowValue: flowDraftValueForPreview(flow.flow), stepId: args.stepId, - args: testArgs, workspace, toolCallbacks, toolId, - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), loadScript: loadScriptForFlowStep, - loadFlowPreviewValue: loadDraftFlowPreviewValue + loadSubflow: loadSubflowForFlowStep }) + + // The module resolution landed on, not the id that was asked for: the job's entrypoint + // override reads the same value, and a form built for the other entrypoint offers fields + // the run will not take. + const isPreprocessor = resolved.module.id === SPECIAL_MODULE_IDS.PREPROCESSOR + // The step's own inputs, never the flow's: a step is fed by its input transforms, so the + // flow's schema names arguments this job would ignore and omits the ones it takes. + const schema = + resolved.code != undefined && resolved.lang + ? await schemaForTestRun({ + content: resolved.code, + language: resolved.lang, + schema: resolved.schema, + entrypoint: isPreprocessor ? 'preprocessor' : undefined + }) + : (resolved.schema ?? {}) + + const stepSummary = resolved.module.summary + return runThroughForm( + { + // The flow: a step has no path of its own, and this is what the status and cancel + // lines quote back, so it has to name something the reader can go and open. The + // step itself is named by the summary below. + path: args.path, + schema, + summary: stepSummary ? `step "${args.stepId}": ${stepSummary}` : `step "${args.stepId}"`, + kind: 'test', + code: resolved.code ?? schema['x-windmill-dyn-select-code'], + lang: resolved.lang ?? schema['x-windmill-dyn-select-lang'], + // Never "deployed": a step test previews the draft flow, and the step's target may + // itself be a draft. + schemaNoun: 'step', + toolName: 'test_run_step', + proposed: args.args, + startMessage: resolved.startMessage, + contextName: resolved.runnableKind, + noun: 'step', + label: `step ${args.stepId}`, + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: resolved.startJob + }, + ctx + ) } async function initApp( @@ -6005,7 +6702,7 @@ async function testRunAppRunnable( toolId, startMessage: `Running backend runnable "${key}" of app "${path}"...`, // A path runnable pointing at a flow really does queue a flow job, so the - // failure path can offer get_flow_run_details; everything else is a script job. + // failure path can offer get_run's step tree; everything else is a script job. contextName: runnable.runType === 'flow' ? 'flow' : 'script', completionName: 'backend runnable', background: args.background, @@ -7389,6 +8086,27 @@ async function deployDraft( `not working until they are.` } + // `policy` is the mode being written, so the exposure is stated from the value in + // hand. What a runnable runs as is the server's to decide, so the note names who + // can reach the app and never an identity. + if (policy.execution_mode === 'anonymous') { + deployNote = + `${deployNote ? `${deployNote} ` : ''}This app is deployed as anonymous: its ` + + `backend runnables are now reachable by anyone with the URL, without logging in. ` + + `Tell the user plainly what is now reachable and by whom.` + } else if (policy.execution_mode === 'guest') { + // Only where the guest door actually opens: below that, silence — a false note + // is worse than none. The standing cap is a live count no read settles, so the + // note says the door is open, not that every newcomer gets in. + if (await guestAccessIsLive(workspace)) { + deployNote = + `${deployNote ? `${deployNote} ` : ''}This app is deployed as guest: anyone ` + + `the instance's identity provider authenticates can now open it and run its ` + + `backend runnables, member of this workspace or not. Tell the user plainly ` + + `what is now reachable and by whom.` + } + } + toolCallbacks.setToolStatus(toolId, { content: `Bundling app "${path}"...` }) @@ -7762,9 +8480,10 @@ export function getActiveGlobalEditorContext( ): GlobalActiveEditorContext | undefined { for (const { itemKind, type } of ACTIVE_GLOBAL_EDITOR_DRAFTS) { const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace }) - const path = liveDraft?.effectivePath || liveDraft?.storagePath + if (!liveDraft) continue + const path = liveDraft.effectivePath || liveDraft.storagePath if (!path) continue - return { type, path, isLiveDraft: true } + return { type, path, storagePath: liveDraft.storagePath, isLiveDraft: true } } } diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts index 4b0dee5de0..7b2ddeb5fc 100644 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/flowRunTree.test.ts @@ -190,4 +190,29 @@ describe('shapeFlowRunTree', () => { expect(parsed.steps[1].result).toBeUndefined() expect(parsed.steps[1].result_total_chars).toBe(700) }) + + it('spends the tree budget on the tree, not on the run payloads carried with it', () => { + const entries = [root()] + for (let i = 1; i <= 12; i++) { + entries.push( + entry({ + step_path: `s${i}`, + flow_step_id: `s${i}`, + status: 'failure', + success: false, + result_prefix: 'y'.repeat(700), + result_length: 700 + }) + ) + } + const logs = 'L'.repeat(12000) + const rendered = shapeFlowRunTree({ entries }, { status: 'failure', logs }) + + // Counting the overrides against the budget would shrink this tree away + // even though the tree itself fits: the model asked for the payloads. + expect(rendered.length).toBeGreaterThan(20000) + const parsed = JSON.parse(rendered) + expect(parsed.run.logs).toBe(logs) + expect(parsed.steps[0].result.length).toBe(700) + }) }) diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts index 207dca1789..af203e2c8c 100644 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts +++ b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts @@ -1,19 +1,25 @@ -import { JobService, type GetFlowAllResultsResponse } from '$lib/gen' +import { JobService, type Job, type GetFlowAllResultsResponse } from '$lib/gen' +import { isWindmillTooBigObject } from '$lib/components/job_args' /** - * Model-facing view of a flow run's execution tree for the global chat's - * get_flow_run_details tool. The backend endpoint (get_flow_all_results) - * enumerates every job of the tree with per-entry truncated results; this - * module shapes that flat list into a compact per-step tree the model can - * read in one tool result. Step addresses ('b/c', 'b[12]/c') are resolved - * server-side by the same endpoint for full-result drill-down. + * Model-facing view of a run for the global chat's get_run tool: the job's own + * summary, args, result and logs, plus — when the run has steps — its execution + * tree. The backend endpoint (get_flow_all_results) enumerates every job of the + * tree with per-entry truncated results; this module shapes that flat list into + * a compact per-step tree the model can read in one tool result. Step addresses + * ('b/c', 'b[12]/c') are resolved server-side by the same endpoint for + * full-result drill-down. + * + * `summarizeRun` also backs list_runs' per-job summary, so it lives here rather + * than in core.ts: get_run needs it, and importing it back would be circular. */ export type FlowResultEntry = GetFlowAllResultsResponse['entries'][number] /** Per-entry result budget requested from the server for the tree view. */ export const TREE_RESULT_HEAD_CHARS = 700 -/** Cap on a drilled single-step full result handed to the model. */ +/** Cap on a full payload handed to the model: a drilled step result, and the + * run's own args, result and logs. */ export const STEP_RESULT_MAX_CHARS = 12000 /** Cap on the whole rendered tree; heads shrink progressively to fit. */ const TREE_TOTAL_BUDGET_CHARS = 20000 @@ -97,6 +103,13 @@ function sliceCodePointSafe(s: string, maxUnits: number): string { return last >= 0xd800 && last <= 0xdbff ? cut.slice(0, -1) : cut } +function sliceCodePointSafeEnd(s: string, maxUnits: number): string { + const cut = s.slice(-maxUnits) + const first = cut.charCodeAt(0) + // drop a leading lone low surrogate + return first >= 0xdc00 && first <= 0xdfff ? cut.slice(1) : cut +} + function shapeResult( entry: FlowResultEntry, opts: ShapeOpts @@ -211,7 +224,8 @@ function shapeChildren(children: FlowTreeNode[], opts: ShapeOpts): Record ): Record { const run = shapeStep(root, opts) const steps = run.steps @@ -221,19 +235,41 @@ function renderTree( if (!steps && root.entry.kind !== 'flow' && root.entry.kind !== 'flowpreview') { run.label = `Job (${root.entry.kind})` } + if (runOverrides) { + // A replacement result supersedes the tree entry's head, and the stale + // truncation marker must go with it. Without one the head stays: it is a + // real server-side prefix of a payload `getJob` would only hand back as a + // WINDMILL_TOO_BIG placeholder. + if ('result' in runOverrides) { + delete run.result + delete run.result_total_chars + } + Object.assign(run, runOverrides) + } return { ...(rootJobNote ? { note: rootJobNote } : {}), run, - ...(steps ? { steps } : {}), - hint: `Results are truncated. Call get_flow_run_details again with step="" (e.g. "b/c", or "b[12]" for one loop iteration) for a step's result in full (up to ${STEP_RESULT_MAX_CHARS} chars).` + ...(steps + ? { + steps, + hint: `Step results are truncated. Call get_run again with step="" (e.g. "b/c", or "b[12]" for one loop iteration) for a step's result in full (up to ${STEP_RESULT_MAX_CHARS} chars), or with id set to a step's job_id for that step's own args, result and logs.` + } + : {}) } } /** Render the whole tree, shrinking result heads until it fits the budget. */ -export function shapeFlowRunTree(response: GetFlowAllResultsResponse): string { +export function shapeFlowRunTree( + response: GetFlowAllResultsResponse, + runOverrides?: Record +): string { const root = buildFlowTree(response.entries) if (!root) { - return 'No jobs found for this run.' + // The tree is the optional half: what the caller already read of the job + // still answers the question, so don't drop it with the missing tree. + return runOverrides + ? JSON.stringify({ run: runOverrides }, null, 1) + : 'No jobs found for this run.' } const notes = [ ...(response.enclosing_job @@ -252,34 +288,193 @@ export function shapeFlowRunTree(response: GetFlowAllResultsResponse): string { ] const rootJobNote = notes.length > 0 ? notes.join(' ') : undefined + // The budget caps the tree, not the run's own args/result/logs — those are + // capped on their own and the model asked for them, so they don't count here + // and the last-resort slice keeps room for them (`run` precedes `steps`). + const overrideChars = runOverrides ? JSON.stringify(runOverrides, null, 1).length : 0 let rendered = '' for (const opts of SHRINK_LADDER) { - rendered = JSON.stringify(renderTree(root, rootJobNote, opts), null, 1) - if (rendered.length <= TREE_TOTAL_BUDGET_CHARS) { + rendered = JSON.stringify(renderTree(root, rootJobNote, opts, runOverrides), null, 1) + if (rendered.length - overrideChars <= TREE_TOTAL_BUDGET_CHARS) { return rendered } } return ( - rendered.slice(0, TREE_TOTAL_BUDGET_CHARS) + + rendered.slice(0, TREE_TOTAL_BUDGET_CHARS + overrideChars) + `\n… (tree truncated at ${TREE_TOTAL_BUDGET_CHARS} chars — drill into specific steps with the step parameter)` ) } -/** Entry point of the get_flow_run_details tool. Without `step`: the compact - * tree. With `step`: that job's full (capped) result, resolved server-side. */ -export async function getFlowRunDetails( - workspace: string, - id: string, - step?: string -): Promise { +// Compact metadata for one run. The raw Job carries args/result/logs/raw_code +// which can be huge — this is only what's needed to identify a run, so list_runs +// can return one entry per job. +export function summarizeRun(job: Job): Record { + const base = { + id: job.id, + job_kind: job.job_kind, + path: job.script_path, + created_by: job.created_by, + created_at: job.created_at, + started_at: job.started_at, + schedule_path: job.schedule_path, + is_flow_step: job.is_flow_step, + tag: job.tag, + worker: job.worker + } + if ('success' in job) { + // CompletedJob. `success` is true for a skipped job too, so is_skipped has + // to be read first or a skipped step reports as a successful one. + return { + ...base, + status: job.canceled + ? 'canceled' + : job.is_skipped + ? 'skipped' + : job.success + ? 'success' + : 'failure', + duration_ms: job.duration_ms + } + } + // QueuedJob. A running job with suspends outstanding is parked — on an approval + // step or on a parallelism slot — not working, and `running` alone hides that. + return { + ...base, + status: job.running ? (job.suspend ? 'suspended' : 'running') : 'queued' + } +} + +/** Cap a payload to STEP_RESULT_MAX_CHARS. `tail` keeps the end instead of the + * start — for logs, where the failure is at the bottom. + * + * Counts and cuts without materialising the string: logs arrive whole and + * unbounded, and spreading one into an array of code points costs ~9x its size + * (a 10MB log measured +90MB). Cutting in UTF-16 units keeps at most the budget + * in code points, never more, so an astral-heavy payload is trimmed slightly + * shorter than advertised rather than overshooting. */ +function cap(text: string, tail = false): string { + // UTF-16 length is never below the code-point count, so anything passing this + // is already under the cap and needs no counting pass at all. + if (text.length <= STEP_RESULT_MAX_CHARS) return text + const total = countCodePoints(text) + if (total <= STEP_RESULT_MAX_CHARS) return text + const note = `… (truncated: ${total} chars total)` + return tail + ? `${note}\n${sliceCodePointSafeEnd(text, STEP_RESULT_MAX_CHARS)}` + : `${sliceCodePointSafe(text, STEP_RESULT_MAX_CHARS)}\n${note}` +} + +/** Told apart from an empty log so the model doesn't report "no logs" for a run + * whose logs it simply failed to read. */ +const LOGS_UNREADABLE = 'Logs could not be read for this run.' + +/** `getJob` swaps a payload over ~90KB for a marker rather than sending it + * (`get_job_query!` in backend/windmill-api/src/jobs.rs), and the two fields use + * different ones: a result becomes the bare string, args become exactly + * `{reason: }`. Each field matches only its own form, so a payload that + * merely carries that string in a `reason` of its own stays the run's value. */ +const TOO_BIG_RESULT = 'WINDMILL_TOO_BIG' + +function stringify(value: unknown): string { + return typeof value === 'string' ? value : JSON.stringify(value, null, 1) +} + +/** An elided args/result payload is reported, never fetched around: the + * endpoints that return those whole take no length parameter, so recovering a + * usable head would mean pulling the entire payload (up to MAX_RESULT_SIZE_MB, + * 500 by default) into the tab to keep 12k of it. The flag is the whole report — + * the result keeps the head the step tree already carries beside it. */ +function shapeRunArgs(job: Job): Record { + if (!job.args) return {} + return isWindmillTooBigObject(job.args) + ? { args_truncated: true } + : { args: cap(stringify(job.args)) } +} + +function shapeRunResult(job: Job): Record { + if (!('result' in job) || job.result === undefined) return {} + // No `result` key when elided, so the tree's `result_prefix` — a real + // server-side head of the same payload — stands in its place unoverridden. + return job.result === TOO_BIG_RESULT + ? { result_truncated: true } + : { result: cap(stringify(job.result)) } +} + +/** Why a run ended badly. Kept out of summarizeRun, which list_runs pays per job. */ +function diagnoseRun(job: Job): Record { + return { + ...(job.canceled_by ? { canceled_by: job.canceled_by } : {}), + ...(job.canceled_reason ? { canceled_reason: job.canceled_reason } : {}), + ...(job.mem_peak ? { mem_peak_kb: job.mem_peak } : {}) + } +} + +/** What get_run answers with: the model's payload, and — when the call addresses one + * job — that job's id, which the card renders the run from. The two are separate + * audiences: `text` is capped for the model, `jobId` is how the user gets the whole + * thing. An address resolving to several jobs (a loop's `b`), an unfinished step or an + * unknown one carries no id, and the call renders as an ordinary tool row. */ +export type RunInspection = { text: string; jobId?: string } + +/** Entry point of the get_run tool. Without `step`: the run's summary, args, + * result and logs, plus the per-step tree when the run has steps. With `step`: + * that step's full (capped) result, resolved server-side. */ +export async function getRun(workspace: string, id: string, step?: string): Promise { if (!step) { - return shapeFlowRunTree( - await JobService.getFlowAllResults({ workspace, id, maxResultLen: TREE_RESULT_HEAD_CHARS }) - ) + // Only the job read is load-bearing: logs and the step tree each answer + // part of the question, so neither failing should cost the model the rest. + const [job, logs, results] = await Promise.all([ + JobService.getJob({ workspace, id, noLogs: true, noCode: true }), + // The dedicated endpoint rather than the job's own `logs` field, which holds too + // little to serve a tail — RunScriptCard's fetch has the mechanism. Only this + // job's own logs either way: a flow's are its orchestration lines, not its steps'. + JobService.getJobLogs({ + workspace, + id, + // Suppress the "to remove ansi colors, use: sed ..." hint the backend + // otherwise prepends — noise for the model, and not actual stripping. + removeAnsiWarnings: true + }).catch(() => LOGS_UNREADABLE), + // `.catch` alone would leave the tree half-guarded: like the log read, this + // one can fail by resolving `undefined` rather than rejecting, and reading + // `.entries` off that throws — losing the job and logs already in hand. + JobService.getFlowAllResults({ workspace, id, maxResultLen: TREE_RESULT_HEAD_CHARS }) + .catch(() => undefined) + .then((r) => r ?? ({ entries: [] } as GetFlowAllResultsResponse)) + ]) + const { id: _id, ...summary } = summarizeRun(job) + const payloads = { ...shapeRunArgs(job), ...shapeRunResult(job) } + // A read that fails without rejecting still arrives here: the generated client + // resolves `undefined` when it cannot read the body (a truncated response, a + // dropped connection). Logs are the one payload where empty is a real answer, + // so that has to be told apart from an empty log rather than reported as one. + const shapedLogs = + typeof logs !== 'string' || logs === LOGS_UNREADABLE + ? LOGS_UNREADABLE + : logs.trim() + ? cap(logs, true) + : 'No logs for this run.' + return { + text: shapeFlowRunTree(results, { + ...summary, + ...diagnoseRun(job), + // A successful read always carries the job itself as the root entry, so + // no entries means the read failed — and nothing else would name the run. + ...(results.entries.length === 0 ? { job_id: id, steps_unavailable: true } : {}), + ...payloads, + logs: shapedLogs + }), + jobId: id + } } - // Drill-down: the server resolves the address directly (a few indexed - // lookups, no tree enumeration) and returns the single job as an entry. + return getStepResult(workspace, id, step) +} + +/** One step's result in full, addressed by step path. The server resolves the + * address directly (a few indexed lookups, no tree enumeration) and returns the + * single job as an entry. */ +async function getStepResult(workspace: string, id: string, step: string): Promise { const response = await JobService.getFlowAllResults({ workspace, id, @@ -287,27 +482,38 @@ export async function getFlowRunDetails( step }) if (response.step_error) { - return ( - response.step_error + - (response.scope_filtered - ? ' (Steps running on tags outside your token’s scope are hidden.)' - : '') - ) + return { + text: + response.step_error + + (response.scope_filtered + ? ' (Steps running on tags outside your token’s scope are hidden.)' + : '') + } } const entry = response.entries[0] if (!entry) { - return 'No jobs found for this run.' + return { text: 'No jobs found for this run.' } } if (entry.status === 'running' || entry.status === 'queued' || entry.status === 'suspended') { - return `Step "${step}" (job ${entry.job_id}) has not completed yet — status: ${entry.status}.` + return { + text: `Step "${step}" (job ${entry.job_id}) has not completed yet — status: ${entry.status}.` + } } + // Every completed step is a job of its own, so the card renders it from source — + // including a skipped one, whose inputs and logs are all there is to see. if (entry.result_prefix === undefined || entry.result_prefix === null) { - return `Step "${step}" (job ${entry.job_id}, ${entry.status}) has no recorded result.` + return { + text: `Step "${step}" (job ${entry.job_id}, ${entry.status}) has no recorded result.`, + jobId: entry.job_id + } } const total = entry.result_length ?? countCodePoints(entry.result_prefix) const capped = total > countCodePoints(entry.result_prefix) ? entry.result_prefix + `\n… (result truncated: ${total} chars total)` : entry.result_prefix - return `Step "${step}" (job ${entry.job_id}, ${entry.status}) result:\n${capped}` + return { + text: `Step "${step}" (job ${entry.job_id}, ${entry.status}) result:\n${capped}`, + jobId: entry.job_id + } } diff --git a/frontend/src/lib/components/copilot/chat/global/gate.ts b/frontend/src/lib/components/copilot/chat/global/gate.ts index 3321d5a0cd..8fb650aa36 100644 --- a/frontend/src/lib/components/copilot/chat/global/gate.ts +++ b/frontend/src/lib/components/copilot/chat/global/gate.ts @@ -9,8 +9,8 @@ * * When the beta ends, replace every call to `isGlobalAiEnabled()` with `true` * and delete this file. The references are intentionally narrow (chat mode - * visibility, custom prompt settings, the `change_mode` tool enum, and the - * AI skills workspace settings tab) so the rip-out is a small grep. + * visibility, custom prompt settings, and the `change_mode` tool enum) so the + * rip-out is a small grep. */ import { logFeatureUsage } from '$lib/utils/featureUsage' diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts index a155115d28..30a4403746 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts @@ -4,7 +4,7 @@ const { getMcpToolsMock, callMcpToolMock, listResourceMock, session } = vi.hoist getMcpToolsMock: vi.fn(), callMcpToolMock: vi.fn(), listResourceMock: vi.fn(), - session: { email: 'first@windmill.dev' } + session: { email: 'first@windmill.dev', workspace_id: 'test-ws', username: 'hugo', pgroups: [] } })) vi.mock('../shared', () => ({ @@ -127,6 +127,12 @@ describe('read/write split', () => { expect(getTool('call_mcp_write_tool').requiresConfirmation).toBe(true) }) + it('admits search and reads in plan mode, never writes', () => { + expect(getTool('search_mcp_tools').planModeSafe).toBe(true) + expect(getTool('call_mcp_read_tool').planModeSafe).toBe(true) + expect(getTool('call_mcp_write_tool').planModeSafe).toBeFalsy() + }) + // The rejection sends the model to the write tool, which classifies from the // same cached listing: without dropping it, that retry is refused too and the // model has nowhere to go until the entry expires. @@ -228,7 +234,7 @@ describe('call results', () => { }) describe('search_mcp_tools', () => { - it('returns compact summaries without the full input schemas', async () => { + it('returns each match with its full input schema', async () => { const result = await run('search_mcp_tools', { query: 'issue' }) expect(result.matches).toEqual([ { @@ -236,7 +242,7 @@ describe('search_mcp_tools', () => { tool: 'get_issue', description: 'Get details of a GitHub issue', mode: 'read', - params: ['owner', 'repo'] + inputSchema: TOOLS[0].inputSchema } ]) }) @@ -330,6 +336,64 @@ describe('result size cap', () => { expect(callMcpToolMock).not.toHaveBeenCalled() }) + it('drops the lowest-ranked schemas before dropping the matches themselves', async () => { + getMcpToolsMock.mockResolvedValue( + ['issue_a', 'issue_b', 'issue_c'].map((name) => ({ + name, + description: 'An issue tool', + inputSchema: { + type: 'object', + properties: { body: { type: 'string', description: 'x'.repeat(9_000) } } + }, + annotations: { readOnlyHint: true } + })) + ) + const result = await run('search_mcp_tools', { query: 'issue' }) + + expect(result.matches.map((m: any) => m.tool)).toEqual(['issue_a', 'issue_b', 'issue_c']) + expect(result.matches[0].inputSchema).toBeDefined() + expect(result.matches[2].inputSchema).toBeUndefined() + expect(result.truncated).toBe(true) + }) + + // Truncating must not swallow the count of matches the score cut off: those + // tools are not in the result at all, so the model has to know to ask again. + it('keeps the more-matches hint when truncation strips schemas', async () => { + getMcpToolsMock.mockResolvedValue( + Array.from({ length: 12 }, (_, i) => ({ + name: `issue_${i}`, + description: 'An issue tool', + inputSchema: { + type: 'object', + properties: { body: { type: 'string', description: 'x'.repeat(3_000) } } + }, + annotations: { readOnlyHint: true } + })) + ) + const result = await run('search_mcp_tools', { query: 'issue' }) + + expect(result.matches).toHaveLength(10) + expect(result.note).toContain('2 more match(es)') + expect(result.note).toContain('inputSchema') + }) + + it('holds the cap when every server failed and nothing matched', async () => { + getMcpToolsMock.mockRejectedValue(new Error('x'.repeat(500))) + const servers = Array.from({ length: 50 }, (_, i) => ({ path: `u/hugo/mcp_${i}` })) + const raw = await createMcpTools(servers) + .find((t) => t.def.function.name === 'search_mcp_tools')! + .fn({ + args: { query: 'issue' }, + workspace: 'test-ws', + helpers: {}, + toolCallbacks: createToolCallbacks(), + toolId: 'tool-1' + }) + + expect(raw.length).toBeLessThanOrEqual(20_000) + expect(JSON.parse(raw).unavailableCount).toBe(50) + }) + it('truncates an oversized tools/list failure in search', async () => { getMcpToolsMock.mockRejectedValue(new Error('x'.repeat(80_000))) const result = await run('search_mcp_tools', { query: 'issue' }) diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts index d5f16d5c54..90afa0b04b 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts @@ -2,14 +2,14 @@ import { z } from 'zod' import { ResourceService, type GetMcpToolsResponse } from '$lib/gen' import { createToolDef, type Tool } from '../shared' import { enabledMcpPaths } from '$lib/components/mcp/enabledServers' +import { isOwnOrSharedMcpPath, MCP_LIST_PER_PAGE, mcpViewer } from '$lib/components/mcp/ownServers' /** * Access to the MCP servers the user has connected (resources of type `mcp`) * as three static tools — search, read call, write call — instead of one * registered tool per remote tool. A server like GitHub's exposes ~90 tools, * whose schemas would otherwise be re-sent on every chat iteration; here only - * matched summaries enter the model's context, and a full input schema only - * after a call fails. + * the tools a search matched enter the model's context, with their schemas. */ type McpToolDef = GetMcpToolsResponse[number] @@ -17,7 +17,7 @@ type McpToolDef = GetMcpToolsResponse[number] export type McpServer = { path: string; editedAt?: string } const MAX_SEARCH_RESULTS = 10 -const MAX_DESCRIPTION_CHARS = 200 +const MAX_DESCRIPTION_CHARS = 1_000 const MAX_RESULT_CHARS = 20_000 // A server writes its own error text, and every enabled server can contribute // one, so search results are capped the same way call results are. @@ -92,13 +92,18 @@ export async function loadMcpServers(workspace: string): Promise { const enabled = enabledMcpPaths(workspace) if (enabled.length === 0) return [] try { - const resources = await ResourceService.listResource({ - workspace, - resourceType: 'mcp', - perPage: 100 - }) + const [resources, viewer] = await Promise.all([ + ResourceService.listResource({ + workspace, + resourceType: 'mcp', + perPage: MCP_LIST_PER_PAGE + }), + mcpViewer(workspace) + ]) return resources - .filter((r) => enabled.includes(r.path)) + .filter( + (r) => enabled.includes(r.path) && isOwnOrSharedMcpPath(r.path, r.extra_perms, viewer) + ) .map((r) => ({ path: r.path, editedAt: r.edited_at })) } catch (e) { console.error('Failed to load MCP servers', e) @@ -139,24 +144,17 @@ function isReadOnly(tool: McpToolDef): boolean { return tool.annotations?.readOnlyHint === true } -function schemaPropertyNames(schema: unknown): string[] { - const properties = (schema as { properties?: Record } | null | undefined) - ?.properties - return properties ? Object.keys(properties) : [] -} - function truncate(text: string, max: number): string { return text.length > max ? text.slice(0, max) + '…' : text } function summarizeTool(server: McpServer, tool: McpToolDef) { - const params = schemaPropertyNames(tool.inputSchema) return { server: server.path, tool: tool.name, description: truncate(tool.description ?? '', MAX_DESCRIPTION_CHARS), mode: isReadOnly(tool) ? 'read' : 'write', - ...(params.length > 0 ? { params } : {}) + inputSchema: tool.inputSchema } } @@ -285,28 +283,44 @@ function bounded(payload: { } /** - * The search payload under the same ceiling as a call result. Server error text - * goes first: the matches are what the model asked for. + * The search payload under the same ceiling as a call result. Schemas go before + * matches: a tool listed without its schema is still callable, since a rejected + * call returns the schema, while a tool dropped from the list is not. */ function boundedSearch(payload: { matches: unknown[]; [k: string]: unknown }): string { - const { unavailable, ...rest } = payload - const dropped = Array.isArray(unavailable) ? { unavailableCount: unavailable.length } : {} - let out = JSON.stringify(payload, null, 2) + let out = JSON.stringify(payload) if (out.length <= MAX_RESULT_CHARS) return out - const matches = [...payload.matches] - const build = () => - JSON.stringify( - { - ...rest, - ...dropped, - matches, - truncated: true, - note: `Truncated to ${MAX_RESULT_CHARS} characters. Refine the query.` - }, - null, - 2 - ) + + // The caller's note says how many matches the score cut off, which truncating + // only adds to. + const { unavailable, note: scoreNote, ...rest } = payload + const dropped = Array.isArray(unavailable) ? { unavailableCount: unavailable.length } : {} + const found = payload.matches.length + const matches = payload.matches.map((m) => ({ ...(m as object) })) as Record[] + const build = () => { + const schemaless = matches.filter((m) => !m.inputSchema).length + const note = typeof scoreNote === 'string' ? [scoreNote] : [] + note.push(`Truncated to ${MAX_RESULT_CHARS} characters.`) + if (schemaless > 0) { + note.push( + `${schemaless} lower-ranked match(es) are listed without their inputSchema; calling one returns the schema if the arguments are wrong.` + ) + } + if (matches.length < found) { + note.push(`${found - matches.length} match(es) dropped — refine the query to see them.`) + } + return JSON.stringify({ ...rest, ...dropped, matches, truncated: true, note: note.join(' ') }) + } + // Dropping the server error text is the first reduction, and the only one left + // when nothing matched. out = build() + if (out.length <= MAX_RESULT_CHARS) return out + // `matches` is ordered by score, so the tail is what the query matched least. + for (let i = matches.length - 1; i >= 0; i--) { + delete matches[i].inputSchema + out = build() + if (out.length <= MAX_RESULT_CHARS) return out + } while (out.length > MAX_RESULT_CHARS && matches.length > 0) { matches.pop() out = build() @@ -331,8 +345,9 @@ const callMcpToolSchema = z.object({ /** * The read and write call tools differ only in which side of the `readOnlyHint` - * split they accept, and that check is what keeps a mutating call behind the - * user's confirmation — building both from one body keeps them from drifting. + * split they accept, and that check is what keeps a mutating call out of plan + * mode and behind the user's confirmation — building both from one body keeps + * them from drifting. */ function createCallTool(servers: McpServer[], mode: 'read' | 'write'): Tool<{}> { const isRead = mode === 'read' @@ -346,13 +361,21 @@ function createCallTool(servers: McpServer[], mode: 'read' | 'write'): Tool<{}> ), showDetails: true, ...(isRead - ? {} + ? { planModeSafe: true } : { requiresConfirmation: true, confirmationMessage: (args: any) => `Call ${args?.tool ?? ''} on ${args?.server ?? ''}` }), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = callMcpToolSchema.parse(args) + // Marks the row with the provider it reaches. Recorded before the listing is + // awaited, so an unreachable server still marks its own failure row, and taken + // from the connected list rather than from `parsed.server`, which is a path the + // model could point anywhere. + const named = servers.find((s) => s.path === parsed.server) + if (named) { + toolCallbacks.setToolStatus(toolId, { mcpServer: { workspace, path: named.path } }) + } // Listing is a live call to a third party: a server that has gone away // must fail this tool, not the chat loop around it. let resolved: Awaited> @@ -405,8 +428,9 @@ export function createMcpTools(servers: McpServer[]): Tool<{}>[] { def: createToolDef( searchMcpToolsSchema, 'search_mcp_tools', - 'Search the tools exposed by the MCP servers connected to this workspace (listed in the system prompt). Returns server + tool names to pass to call_mcp_read_tool or call_mcp_write_tool.' + 'Search the tools exposed by the MCP servers connected to this workspace (listed in the system prompt). Returns server + tool names to pass to call_mcp_read_tool or call_mcp_write_tool, each with the input schema its arguments must follow.' ), + planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = searchMcpToolsSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: 'Searching MCP tools...' }) diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index a6c1318638..ca43598d26 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -141,6 +141,7 @@ function scriptDraftToWorkspaceItem(path: string, draft: NewScript): WorkspaceIt summary: draft.summary, language: draft.language, value: draft.content, + schema: draft.schema, parentHash: draft.parent_hash, isDraft: true } @@ -175,6 +176,9 @@ function appDraftToWorkspaceItem(path: string, draft: AppDraftValue): WorkspaceI summary: value.summary, parentVersionId: value.parent_version, value, + // The chat only ever addresses the `raw_app` draft kind (see itemKindFor), + // so every app draft it can see is a code app. + rawApp: true, isDraft: true } } @@ -541,6 +545,7 @@ function backendDraftRowToWorkspaceItem( value: undefined, isDraft: true, triggerKind, + rawApp: row.kind === 'raw_app' ? true : undefined, ...(isLiveDraft ? { isLiveDraft: true } : {}) } } diff --git a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts index 8a9978ecbc..a60f7bd79f 100644 --- a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts +++ b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts @@ -140,6 +140,18 @@ export type WorkspaceItem = { * without it a reader cannot tell a secret from a plain variable, since the * value is always redacted. */ isSecret?: boolean + /** Apps only. True for a code app, false for one built in the drag-and-drop + * editor. The two are edited by disjoint tool sets, so the distinction has to + * reach the model before it picks one. */ + rawApp?: boolean + /** Apps only. Who may open the app — read from a draft, who may open it once + * that draft is deployed: `anonymous` is anyone with the URL and no login, + * `guest` anyone the identity provider authenticates, the rest a workspace + * member. A `guest` app whose instance or workspace admits + * no guest says so here, since the mode is stored either way. Reported per + * app, not in listings — a listing carrying it would make absence read as + * proof of no exposure. */ + executionMode?: string isDraft: boolean isLiveDraft?: boolean } diff --git a/frontend/src/lib/components/copilot/chat/itemPreview.ts b/frontend/src/lib/components/copilot/chat/itemPreview.ts new file mode 100644 index 0000000000..ef9f14ddc9 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/itemPreview.ts @@ -0,0 +1,30 @@ +// The session preview panel's action, kept out of `shared.ts` so the chat message render +// path can import it at runtime without pulling in that module's graph and risking the +// chunk cycles docs/frontend-import-cycles.md exists to prevent. Keep this file import-free. + +/** Item kinds a session preview can host: the three live editors, which are also the + * subset a write tool can land. */ +export type PreviewCardKind = 'script' | 'flow' | 'raw_app' + +// Dispatched by a preview card on a tool call that created or updated a workspace item, +// and by a path link in a chat message. Opens the item's live editor in the session side +// panel — or focuses the tab if it is already open. The handler is registered by the +// sessions page (the only surface with a preview panel). +export type OpenItemPreviewAction = { + id: string + type: 'open_item_preview' + label: string + previewKind: PreviewCardKind + path: string +} + +/** Build the action a preview card or path link dispatches from its (kind, path). */ +export function openItemPreviewAction(kind: PreviewCardKind, path: string): OpenItemPreviewAction { + return { + id: `open-item-preview:${kind}:${path}`, + type: 'open_item_preview', + label: `Open ${kind === 'raw_app' ? 'app' : kind} preview`, + previewKind: kind, + path + } +} diff --git a/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts b/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts index 264a438d2b..b64711dccf 100644 --- a/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts @@ -1,6 +1,6 @@ /** - * A message draft: the four lanes that ship together with one send — text, - * pastes, images, text files. Every place a draft accumulates or moves + * A message draft: the five lanes that ship together with one send — text, + * pastes, images, text files, blobs. Every place a draft accumulates or moves * (composer attach, queue append, dequeue restore, failure restore) goes * through this type, so the draft rules — file dedupe by source identity, * courtesy rename, attachment slot caps, all-lanes-move-together — live here @@ -10,6 +10,7 @@ * manager-wide state — enforced at the composer until it moves into the * store) and @context/DOM picks (ContextManager owns their lifecycle). */ +import { MAX_ATTACHED_BLOBS, type AttachedBlob } from './blobUtils' import { MAX_ATTACHED_IMAGES, type AttachedImage } from './imageUtils' import type { PasteAttachment } from './pasteTokens' import { @@ -19,12 +20,13 @@ import { type AttachedTextFile } from './textFileUtils' -/** A draft's four lanes as plain data — what moves between owners. */ +/** A draft's five lanes as plain data — what moves between owners. */ export interface DraftSnapshot { text: string pastes: PasteAttachment[] images: AttachedImage[] files: AttachedTextFile[] + blobs: AttachedBlob[] } export class MessageDraft { @@ -32,12 +34,14 @@ export class MessageDraft { pastes = $state([]) images = $state([]) files = $state([]) + blobs = $state([]) constructor(seed?: Partial) { if (seed?.text) this.text = seed.text if (seed?.pastes) this.pastes = [...seed.pastes] if (seed?.images) this.images = [...seed.images] if (seed?.files) this.files = [...seed.files] + if (seed?.blobs) this.blobs = [...seed.blobs] } get isEmpty(): boolean { @@ -45,12 +49,13 @@ export class MessageDraft { this.text.trim() === '' && this.pastes.length === 0 && this.images.length === 0 && - this.files.length === 0 + this.files.length === 0 && + this.blobs.length === 0 ) } get hasAttachments(): boolean { - return this.images.length > 0 || this.files.length > 0 + return this.images.length > 0 || this.files.length > 0 || this.blobs.length > 0 } /** Files joining a draft always fold (dedupe by source identity, courtesy @@ -83,6 +88,14 @@ export class MessageDraft { return dropped } + /** Blobs join up to the slot cap. Returns the dropped count (caller toasts). */ + addBlobs(blobs: AttachedBlob[]): number { + const merged = [...this.blobs, ...blobs] + const dropped = Math.max(0, merged.length - MAX_ATTACHED_BLOBS) + this.blobs = merged.slice(0, MAX_ATTACHED_BLOBS) + return dropped + } + /** * Merge a restored draft on top of this one (queued-message delete, restore * after a cancelled/errored turn): the restored draft was written FIRST, so @@ -91,10 +104,16 @@ export class MessageDraft { * Returns whether text merged onto a non-empty draft (the caller must then * keep both drafts' context), plus dropped counts for toasts. */ - prepend(restored: { text: string; images?: AttachedImage[]; files?: AttachedTextFile[] }): { + prepend(restored: { + text: string + images?: AttachedImage[] + files?: AttachedTextFile[] + blobs?: AttachedBlob[] + }): { mergedIntoDraft: boolean droppedImages: number droppedFiles: number + droppedBlobs: number } { const mergedIntoDraft = !!restored.text && !!this.text.trim() // An attachment-only restore has empty text; prepending would only add blank lines. @@ -115,7 +134,13 @@ export class MessageDraft { droppedFiles = Math.max(0, merged.length - MAX_ATTACHED_FILES) this.files = merged.slice(0, MAX_ATTACHED_FILES) } - return { mergedIntoDraft, droppedImages, droppedFiles } + let droppedBlobs = 0 + if (restored.blobs?.length) { + const merged = [...restored.blobs, ...this.blobs] + droppedBlobs = Math.max(0, merged.length - MAX_ATTACHED_BLOBS) + this.blobs = merged.slice(0, MAX_ATTACHED_BLOBS) + } + return { mergedIntoDraft, droppedImages, droppedFiles, droppedBlobs } } /** Replace the draft with a snapshot, but only when it is empty — an occupied @@ -132,16 +157,18 @@ export class MessageDraft { this.pastes = [...(snapshot.pastes ?? [])] this.images = [...(snapshot.images ?? [])] this.files = [...(snapshot.files ?? [])] + this.blobs = [...(snapshot.blobs ?? [])] } - /** Snapshot and clear atomically — the four lanes always move together, so no + /** Snapshot and clear atomically — the five lanes always move together, so no * call site can take one and forget another. */ take(): DraftSnapshot { const snapshot: DraftSnapshot = { text: this.text, pastes: this.pastes, images: this.images, - files: this.files + files: this.files, + blobs: this.blobs } this.clear() return snapshot @@ -152,5 +179,6 @@ export class MessageDraft { this.pastes = [] this.images = [] this.files = [] + this.blobs = [] } } diff --git a/frontend/src/lib/components/copilot/chat/messageDraft.test.ts b/frontend/src/lib/components/copilot/chat/messageDraft.test.ts index a6a9bb6169..dec8e1e6b9 100644 --- a/frontend/src/lib/components/copilot/chat/messageDraft.test.ts +++ b/frontend/src/lib/components/copilot/chat/messageDraft.test.ts @@ -5,22 +5,49 @@ import { MessageDraft } from './messageDraft.svelte' // the draft-level guarantees: lanes move together, restores respect occupancy, // aggregation always applies the rules. +function blob(name: string) { + return { + name, + mediaType: 'application/pdf', + dataUrl: 'data:application/pdf;base64,JVBERg==', + size: 4 + } +} + describe('MessageDraft', () => { - it('take() snapshots and clears all four lanes atomically', () => { + it('take() snapshots and clears all five lanes atomically', () => { const d = new MessageDraft({ text: 'hello', pastes: [{ id: 'p1', content: 'x' } as any], images: [{ dataUrl: 'i1' } as any], - files: [{ name: 'a.md', content: 'a' }] + files: [{ name: 'a.md', content: 'a' }], + blobs: [blob('a.pdf')] }) const snap = d.take() expect(snap.text).toBe('hello') expect(snap.pastes).toHaveLength(1) expect(snap.images).toHaveLength(1) expect(snap.files).toHaveLength(1) + expect(snap.blobs).toHaveLength(1) expect(d.isEmpty).toBe(true) }) + it('treats a blob-only draft as occupied and caps blobs, keeping restored ones first', () => { + const d = new MessageDraft({ blobs: [blob('only.pdf')] }) + expect(d.isEmpty).toBe(false) + expect(d.hasAttachments).toBe(true) + expect(d.replaceIfEmpty({ text: 'restored' })).toBe(false) + + const dropped = d.addBlobs(Array.from({ length: 8 }, (_, i) => blob(`new${i}.pdf`))) + expect(dropped).toBe(1) + expect(d.blobs).toHaveLength(8) + + const res = d.prepend({ text: '', blobs: [blob('old.pdf')] }) + expect(res.droppedBlobs).toBe(1) + expect(d.blobs[0].name).toBe('old.pdf') + expect(d.blobs).toHaveLength(8) + }) + it('replaceIfEmpty declines when any lane is occupied', () => { const d = new MessageDraft({ files: [{ name: 'a.md', content: 'a' }] }) expect(d.replaceIfEmpty({ text: 'restored' })).toBe(false) diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 7583c1b356..f696b1c869 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. -- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
    \`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure = [where ]\`, \`// dimension = \`. +- \`materialize\` (the managed output): a managed \`// materialize ducklake:///
    \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
    \`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt:////\`, with \`\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt:////\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". - \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Every node of this pipeline lives at \`f/${ctx.folder}/\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`\`). diff --git a/frontend/src/lib/components/copilot/chat/planModeMessages.ts b/frontend/src/lib/components/copilot/chat/planModeMessages.ts index 0b502ba839..85710d0c54 100644 --- a/frontend/src/lib/components/copilot/chat/planModeMessages.ts +++ b/frontend/src/lib/components/copilot/chat/planModeMessages.ts @@ -12,6 +12,9 @@ export const PLAN_MODE_MESSAGES = { /** Sits beside the autonomy picker while plan mode holds. The picker's tooltip carries * the rest, so this states only the constraint. */ modeNote: 'Read-only', + /** Refuses a pending run form. Its own string because nothing is settled: the form stays + * live, so this names the way out rather than telling the user their run was blocked. */ + runFormRefused: 'Plan mode is read-only — switch it off to run this script.', // One pair for both artifact tools: the fact and the way forward are the same whether the // model tried to mint the plan or to rewrite it, and the generic refusal above ("put this // change in your plan") reads as nonsense for a call that writes a document. diff --git a/frontend/src/lib/components/copilot/chat/safeHref.test.ts b/frontend/src/lib/components/copilot/chat/safeHref.test.ts new file mode 100644 index 0000000000..5204ebe994 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { safeHref } from './safeHref' + +const BASE = 'https://app.example.com/sessions?workspace=demo' + +describe('safeHref', () => { + it.each([ + 'https://windmill.dev/docs', + 'http://localhost:3000/', + 'mailto:someone@example.com', + '/runs/abc', + '#anchor', + 'docs/page' + ])('keeps %s', (href) => { + expect(safeHref(href, BASE)).toBe(href) + }) + + it.each([ + 'javascript:alert(1)', + 'JavaScript:alert(1)', + ' javascript:alert(1)', + 'data:text/html,', + 'vbscript:msgbox', + 'file:///etc/passwd' + ])('drops %s', (href) => { + expect(safeHref(href, BASE)).toBeUndefined() + }) + + it('drops a missing or empty href', () => { + expect(safeHref(undefined, BASE)).toBeUndefined() + expect(safeHref('', BASE)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/safeHref.ts b/frontend/src/lib/components/copilot/chat/safeHref.ts new file mode 100644 index 0000000000..3acdabb464 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.ts @@ -0,0 +1,17 @@ +const SAFE_PROTOCOLS = ['http:', 'https:', 'mailto:'] + +/** + * The href a rendered markdown link may carry, or undefined for one that must be dropped. + * + * Markdown reaches the chat renderers from a model, and from another member for a shared + * artifact, and `svelte-exmarkdown` passes `javascript:` and `data:` hrefs through untouched. + * Relative links resolve against `base` (the page), so they stay. + */ +export function safeHref(href: string | undefined, base: string): string | undefined { + if (!href) return undefined + try { + return SAFE_PROTOCOLS.includes(new URL(href, base).protocol) ? href : undefined + } catch { + return undefined + } +} diff --git a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte index 60687efcce..e17cdbcb36 100644 --- a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte @@ -165,7 +165,7 @@ {#snippet settings()} -
    +
    + Role + Privileges + On + + + + + {#each grantRows as grant (grantKey(grant))} + {@const revokeScope = revokeScopeOf(grant)} + {@const revocable = revocablePrivileges(grant, target)} + {@const blocked = blockingSources(grant, revocable)} + {@const uncovered = uncoveredCreators(grant, info.roles)} + + {grant.grantee} + {grant.privileges.join(', ')} + + {grantScopeLabel(grant)} + {#if blocked.length > 0} + + from {blocked.join(', ')} + + {/if} + {#if uncovered.length > 0} + + · not for what {uncovered.join(', ')} + {uncovered.length === 1 ? 'creates' : 'create'} + + {/if} + + + + {#if info.editable && revokeScope && revocable.length > 0 && blocked.length === 0 && info.roles.includes(grant.grantee) && grant.grantee !== ADMIN_ROLE} + + + {/if} + + +{/if} + + (pending = undefined)} +> +
    + {#if pendingCoversObjects} + + The same privileges on several objects read as one row, and are revoked together. + + {/if} + {#each pending?.warnings ?? [] as warning (warning)} + {warning} + {/each} + + Runs against {datatable} in a single transaction: + +
    {(pending?.statements ?? []).join(';\n')};
    +
    +
    diff --git a/frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte b/frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte new file mode 100644 index 0000000000..5e5119abb6 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte @@ -0,0 +1,95 @@ + + +
    +
    + GRANT + ({ value: p, label: p }))} + placeholder="privileges" + {disabled} + size="sm" + class="min-w-48" + /> + ON + ({ value: r, label: r }))} + placeholder="role" + {disabled} + size="sm" + class="w-40" + /> + +
    + {#if statement} +
    {statement}
    + {/if} +
    diff --git a/frontend/src/lib/components/datatableAcl/aclScopes.test.ts b/frontend/src/lib/components/datatableAcl/aclScopes.test.ts new file mode 100644 index 0000000000..7c01836e89 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/aclScopes.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from 'vitest' +import type { AclGrant } from '$lib/gen' +import { + blockingSources, + grantKey, + groupGrants, + revocablePrivileges, + revokeScopeOf, + uncoveredCreators +} from './aclScopes' + +const table = (name: string) => ({ name, kind: 'TABLE' }) +const by = (role: string, privileges: string[], reachable = true) => ({ + role, + privileges, + reachable +}) +const byAdmin = (grant: Omit): AclGrant => ({ + ...grant, + sources: [by('admin', grant.privileges)] +}) + +describe('grantKey', () => { + it('tells apart a table and a function of the same name', () => { + const row = (object: { name: string; kind: string; args?: string }) => ({ + grantee: 'analytics', + privileges: ['SELECT'], + objects: [object], + sources: [by('admin', ['SELECT'])] + }) + expect(grantKey(row(table('orders')))).not.toBe( + grantKey(row({ name: 'orders', kind: 'FUNCTION', args: '' })) + ) + }) +}) + +describe('groupGrants', () => { + // A row's revoke names every object in it, so a row must only hold what one revoke may take. + it('folds the same privileges on objects of one kind, and nothing else', () => { + const grants: AclGrant[] = [ + { grantee: 'analytics', privileges: ['SELECT'], object: table('orders') }, + { grantee: 'analytics', privileges: ['SELECT'], object: table('salaries') }, + { grantee: 'operator', privileges: ['SELECT'], object: table('orders') }, + { grantee: 'analytics', privileges: ['INSERT', 'SELECT'], object: table('events') }, + { grantee: 'analytics', privileges: ['SELECT'], object: { name: 's', kind: 'SEQUENCE' } }, + { grantee: 'analytics', privileges: ['SELECT'], future: 'TABLES' }, + { grantee: 'analytics', privileges: ['USAGE'] } + ].map(byAdmin) + const rows = groupGrants(grants) + expect(rows.map((r) => [r.grantee, r.privileges, r.objects, r.future])).toEqual([ + ['analytics', ['SELECT'], [table('orders'), table('salaries')], undefined], + ['operator', ['SELECT'], [table('orders')], undefined], + ['analytics', ['INSERT', 'SELECT'], [table('events')], undefined], + ['analytics', ['SELECT'], [{ name: 's', kind: 'SEQUENCE' }], undefined], + ['analytics', ['SELECT'], [], 'TABLES'], + ['analytics', ['USAGE'], [], undefined] + ]) + }) + + // A revoke takes the row back from every source, so the row must name them all, with what each + // gave. + it('keeps every source of the grants it folds', () => { + const grants: AclGrant[] = [ + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('orders'), + sources: [by('admin', ['SELECT'])] + }, + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('salaries'), + sources: [by('admin', ['SELECT']), by('operator', ['SELECT'])] + } + ] + expect(groupGrants(grants)[0].sources).toEqual([ + by('admin', ['SELECT']), + by('operator', ['SELECT']) + ]) + }) +}) + +describe('revoke of a row', () => { + const row = (future?: string) => ({ + grantee: 'analytics', + privileges: ['SELECT'], + objects: [], + future, + sources: [by('admin', ['SELECT'])] + }) + + it('takes back only what the editor may revoke on the database', () => { + const database = { ...row(), privileges: ['CONNECT', 'CREATE'] } + expect(revocablePrivileges(database, { kind: 'database' })).toEqual(['CREATE']) + expect(revocablePrivileges(database, { kind: 'schema', schema: 'public' })).toEqual([ + 'CONNECT', + 'CREATE' + ]) + // Set database-wide, so not one a schema's revoke could take back either. + const schemasLater = { ...row('SCHEMAS'), privileges: ['CREATE'] } + expect(revocablePrivileges(schemasLater, { kind: 'database' })).toEqual([]) + }) + + it('maps default privileges to their scope, and refuses the ones it has none for', () => { + expect(revokeScopeOf(row())).toBe('target') + expect(revokeScopeOf(row('TABLES'))).toBe('future_tables') + expect(revokeScopeOf(row('TYPES'))).toBeUndefined() + expect(revokeScopeOf({ ...row(), objects: [{ name: 'mood', kind: 'TYPE' }] })).toBeUndefined() + }) + + // Postgres takes a grant back only through its source: offering the revoke would promise what + // the plan then refuses. But only the sources of what is revoked count: the catalog's CONNECT + // on the database comes from its owner, out of reach, and must not hold back a CREATE the + // editor granted. + it('is held back only by a source out of reach for what it takes', () => { + const database = { + ...row(), + privileges: ['CONNECT', 'CREATE'], + sources: [by('postgres', ['CONNECT'], false), by('admin', ['CREATE'])] + } + const revocable = revocablePrivileges(database, { kind: 'database' }) + expect(blockingSources(database, revocable)).toEqual([]) + expect(blockingSources(database, ['CONNECT'])).toEqual(['postgres']) + const partly = { + ...row('TABLES'), + sources: [by('admin', ['SELECT']), by('postgres', ['SELECT'], false)] + } + expect(blockingSources(partly, ['SELECT'])).toEqual(['postgres']) + }) + + // Whether a grant can be taken back depends on its object, so a row folding several objects is + // only revocable if each of its grants is. + it('is held back by a source out of reach on any of the objects it folds', () => { + const grants: AclGrant[] = [ + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('orders'), + sources: [by('admin', ['SELECT'])] + }, + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('salaries'), + sources: [by('admin', ['SELECT'], false)] + } + ] + const [folded] = groupGrants(grants) + expect(folded.objects).toHaveLength(2) + expect(blockingSources(folded, ['SELECT'])).toEqual(['admin']) + // Folding reads the grants, never rewrites them. + expect(grants[0].sources[0].reachable).toBe(true) + }) +}) + +describe('uncoveredCreators', () => { + // A default privilege binds only the creating roles it was granted for: a role added since is + // left out until the grant is made again. + it('names the roles a created-later row leaves out', () => { + const future = { + grantee: 'analytics', + privileges: ['SELECT'], + objects: [], + future: 'TABLES', + sources: [by('admin', ['SELECT']), by('analytics', ['SELECT'])] + } + expect(uncoveredCreators(future, ['admin', 'analytics', 'late'])).toEqual(['late']) + expect(uncoveredCreators({ ...future, future: undefined }, ['late'])).toEqual([]) + // Set by a role outside the catalog, it was never meant to cover the catalog's roles. + expect( + uncoveredCreators({ ...future, sources: [by('postgres', ['SELECT'], false)] }, [ + 'admin', + 'late' + ]) + ).toEqual([]) + }) +}) diff --git a/frontend/src/lib/components/datatableAcl/aclScopes.ts b/frontend/src/lib/components/datatableAcl/aclScopes.ts new file mode 100644 index 0000000000..d5b8ad9d20 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/aclScopes.ts @@ -0,0 +1,211 @@ +import type { AclGrant, AclSource, AclTarget } from '$lib/gen' + +/** The role a data table connects as without roles — `custom_instance_user` in Postgres. */ +export const ADMIN_ROLE = 'admin' + +/** Privileges Postgres accepts per kind of object. Mirrors the whitelist the backend validates + * against — a privilege missing here just cannot be built. */ +/** `CREATE` on a database is the right to create schemas in it, and the only database privilege + * handed out here: `CONNECT` is managed with the instance's role catalog. */ +export const DATABASE_PRIVILEGES = ['CREATE'] +export const SCHEMA_PRIVILEGES = ['USAGE', 'CREATE'] +export const TABLE_PRIVILEGES = [ + 'SELECT', + 'INSERT', + 'UPDATE', + 'DELETE', + 'TRUNCATE', + 'REFERENCES', + 'TRIGGER' +] +/** Postgres 17 and later only, so it is offered from what the server reports. */ +export const MAINTAIN_PRIVILEGE = 'MAINTAIN' +export const SEQUENCE_PRIVILEGES = ['USAGE', 'SELECT', 'UPDATE'] +export const FUNCTION_PRIVILEGES = ['EXECUTE'] + +export type AclScope = + | 'target' + | 'all_tables' + | 'all_sequences' + | 'all_functions' + | 'future_tables' + | 'future_sequences' + | 'future_functions' + +export type AclTargetKind = AclTarget['kind'] + +/** The scopes a target can grant on, in the order the builder offers them. */ +export function scopesOf(kind: AclTargetKind): { value: AclScope; label: string }[] { + if (kind === 'database') return [{ value: 'target', label: 'the database itself' }] + if (kind === 'table') return [{ value: 'target', label: 'this table' }] + return [ + { value: 'target', label: 'the schema itself' }, + { value: 'all_tables', label: 'all tables in it' }, + { value: 'all_sequences', label: 'all sequences in it' }, + { value: 'all_functions', label: 'all functions in it' }, + { value: 'future_tables', label: 'tables created later' }, + { value: 'future_sequences', label: 'sequences created later' }, + { value: 'future_functions', label: 'functions created later' } + ] +} + +export function privilegesOf( + scope: AclScope, + kind: AclTargetKind, + supportsMaintain = false +): string[] { + const tablePrivileges = supportsMaintain + ? [...TABLE_PRIVILEGES, MAINTAIN_PRIVILEGE] + : TABLE_PRIVILEGES + switch (scope) { + case 'target': + if (kind === 'database') return DATABASE_PRIVILEGES + return kind === 'schema' ? SCHEMA_PRIVILEGES : tablePrivileges + case 'all_tables': + case 'future_tables': + return tablePrivileges + case 'all_sequences': + case 'future_sequences': + return SEQUENCE_PRIVILEGES + case 'all_functions': + case 'future_functions': + return FUNCTION_PRIVILEGES + } +} + +/** What a statement built at this scope reads as, for the builder's own preview. */ +export function scopeSql(scope: AclScope, target: AclTarget, dbname?: string): string { + if (target.kind === 'database') return `DATABASE ${dbname ?? ''}`.trim() + const schema = target.schema + switch (scope) { + case 'target': + return target.kind === 'schema' ? `SCHEMA ${schema}` : `TABLE ${schema}.${target.table}` + case 'all_tables': + return `ALL TABLES IN SCHEMA ${schema}` + case 'all_sequences': + return `ALL SEQUENCES IN SCHEMA ${schema}` + case 'all_functions': + return `ALL FUNCTIONS IN SCHEMA ${schema}` + case 'future_tables': + return `TABLES (default privileges in ${schema})` + case 'future_sequences': + return `SEQUENCES (default privileges in ${schema})` + case 'future_functions': + return `FUNCTIONS (default privileges in ${schema})` + } +} + +/** One row of the grants table: the same privileges on several objects read as one line, since + * granting them per object is what `ON ALL TABLES` does. */ +export type GroupedGrant = { + grantee: string + privileges: string[] + objects: NonNullable[] + future?: string + /** Every role the row's grants come from, each once, with what it gave. */ + sources: AclSource[] +} + +export function groupGrants(grants: AclGrant[]): GroupedGrant[] { + const rows: GroupedGrant[] = [] + for (const grant of grants) { + const existing = grant.object + ? rows.find( + (r) => + r.grantee === grant.grantee && + r.future === grant.future && + r.objects[0]?.kind === grant.object?.kind && + r.privileges.join() === grant.privileges.join() + ) + : undefined + if (existing) { + existing.objects.push(grant.object!) + for (const source of grant.sources) { + const known = existing.sources.find((s) => s.role === source.role) + // Whether a role's grant can be taken back depends on the object it is on, so a row + // holds a source as reachable only if it is on every object the row folds. + if (known) { + known.reachable &&= source.reachable + known.privileges = [...new Set([...known.privileges, ...source.privileges])].sort() + } else { + existing.sources.push({ ...source, privileges: [...source.privileges] }) + } + } + } else { + rows.push({ + grantee: grant.grantee, + privileges: grant.privileges, + objects: grant.object ? [grant.object] : [], + future: grant.future, + sources: grant.sources.map((s) => ({ ...s, privileges: [...s.privileges] })) + }) + } + } + return rows +} + +/** The roles that gave some of `privileges` and that this data table's connection cannot act for. + * Only they can take those grants back, so a revoke of `privileges` is not offered. */ +export function blockingSources(grant: GroupedGrant, privileges: string[]): string[] { + return grant.sources + .filter((s) => !s.reachable && s.privileges.some((p) => privileges.includes(p))) + .map((s) => s.role) +} + +/** Which of `roles` a "created later" row granted for some of them does not cover. A default + * privilege binds only the creating roles it was granted for, so what the others create stays out + * of it. A row none of `roles` set — the instance's own, say — was never meant to cover them, and + * names none. */ +export function uncoveredCreators(grant: GroupedGrant, roles: string[]): string[] { + if (!grant.future || !grant.sources.some((s) => roles.includes(s.role))) return [] + return roles.filter((r) => !grant.sources.some((s) => s.role === r)) +} + +/** A row's identity. Two rows may share a grantee and an object name — a table `orders` and a + * function `orders()` — so the kind and the privileges are part of it too. */ +export function grantKey(grant: GroupedGrant): string { + return [ + grant.grantee, + grant.future ?? '', + grant.privileges.join(','), + ...grant.objects.map((o) => `${o.kind}:${o.name}(${o.args ?? ''})`) + ].join('|') +} + +/** The scope a revoke of this row takes, or `undefined` when the builder cannot express it — + * Postgres also records privileges on types, present and default, which nothing here grants and + * the API has no scope for. */ +export function revokeScopeOf(grant: GroupedGrant): AclScope | undefined { + if (!grant.future) return grant.objects.some((o) => o.kind === 'TYPE') ? undefined : 'target' + const scope = `future_${grant.future.toLowerCase()}` + return (['future_tables', 'future_sequences', 'future_functions'] as const).find( + (s) => s === scope + ) +} + +/** The privileges of a row a revoke may take back. On the database that is `CREATE` alone: + * `CONNECT` belongs to the role catalog, which would grant it again, and `TEMPORARY` is not one + * the editor hands out — a row holding only those has nothing to revoke here. A database's rows + * "created later" are default privileges set database-wide, which nothing here revokes. */ +export function revocablePrivileges(grant: GroupedGrant, target: AclTarget): string[] { + if (target.kind === 'database' && grant.objects.length === 0) { + if (grant.future) return [] + return grant.privileges.filter((p) => DATABASE_PRIVILEGES.includes(p)) + } + return grant.privileges +} + +/** How a row reads back: what it covers, in one phrase. */ +export function grantScopeLabel(grant: GroupedGrant): string { + if (grant.future) return `${grant.future.toLowerCase()} created later` + if (grant.objects.length === 1) { + const object = grant.objects[0] + // A routine's arguments are part of what it is, so two of the same name would otherwise + // read as one row twice. + const args = object.args !== undefined ? `(${object.args})` : '' + return `${object.kind.toLowerCase()} ${object.name}${args}` + } + if (grant.objects.length > 1) + return `${grant.objects.length} ${grant.objects[0].kind.toLowerCase()}s` + return 'itself' +} diff --git a/frontend/src/lib/components/datatableMigrationRole.test.ts b/frontend/src/lib/components/datatableMigrationRole.test.ts new file mode 100644 index 0000000000..6d136b8371 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.test.ts @@ -0,0 +1,60 @@ +import { describe, test, expect } from 'vitest' +import { parseMigrationRole, withMigrationRole } from './datatableMigrationRole' + +describe('parseMigrationRole', () => { + test('reads every spelling the server accepts from the leading comment block', () => { + for (const line of [ + '-- role analyst', + '-- Role: analyst', + '-- role=analyst', + '-- role analyst;' + ]) { + expect(parseMigrationRole(`\n${line}\nBEGIN;\nEND;`)).toEqual({ + kind: 'role', + role: 'analyst' + }) + } + }) + + test('an annotation below BEGIN is not one', () => { + expect(parseMigrationRole('BEGIN;\n-- role analyst\nEND;')).toEqual({ kind: 'none' }) + }) + + test('a malformed attempt is an error, not the default', () => { + for (const line of [ + '-- role based access below', + '-- role', + '-- role:', + '-- role an;alytics' + ]) { + expect(parseMigrationRole(`${line}\nBEGIN;`)).toEqual({ kind: 'malformed', line }) + } + }) + + test('comments that do not start with the word role are ignored', () => { + expect(parseMigrationRole('-- roles analyst\n-- rolex\nBEGIN;')).toEqual({ kind: 'none' }) + }) +}) + +describe('withMigrationRole', () => { + test('leads above BEGIN, so the server reads it', () => { + const out = withMigrationRole('BEGIN;\nSELECT 1;\nEND;', 'analyst') + expect(out).toBe('-- role analyst\nBEGIN;\nSELECT 1;\nEND;') + }) + + test('replaces any attempt rather than stacking, malformed ones included', () => { + const out = withMigrationRole( + '-- Role: auditor\n-- role oops no\n-- keep me\nBEGIN;', + 'analyst' + ) + expect(out).toBe('-- role analyst\n-- keep me\nBEGIN;') + }) + + test('undefined strips the annotation, so it runs as admin', () => { + expect(withMigrationRole('-- role analyst\n\nBEGIN;\nEND;', undefined)).toBe('BEGIN;\nEND;') + }) + + test('refuses a name the server would refuse', () => { + expect(() => withMigrationRole('BEGIN;', 'bad;name')).toThrow() + }) +}) diff --git a/frontend/src/lib/components/datatableMigrationRole.ts b/frontend/src/lib/components/datatableMigrationRole.ts new file mode 100644 index 0000000000..5475bf2fd7 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.ts @@ -0,0 +1,70 @@ +import { isDatatableRoleName } from './dbTypes' + +/** + * A migration carries the data table role it runs as in its own SQL, as a `-- role ` + * annotation. There is no separate field: the annotation is what the server reads, and keeping + * it in the SQL is what lets it survive a `wmill sync` round-trip. + * + * Mirrors `SqlAnnotations::datatable_role` on the backend. It is only read from the leading + * comment block, so an annotation below `BEGIN;` is ignored and the migration runs as admin. A + * leading comment whose first word is `role` is an annotation attempt, and a malformed one is an + * error there, so it is one here too. + */ + +export type MigrationRole = + | { kind: 'none' } + | { kind: 'role'; role: string } + | { kind: 'malformed'; line: string } + +/** The body of a leading comment line that attempts a role annotation, or undefined. */ +function roleAttempt(line: string): string | undefined { + if (!line.startsWith('--')) return undefined + const body = line.slice(2).trimStart() + if (body.slice(0, 4).toLowerCase() !== 'role') return undefined + const after = body.slice(4) + if (after !== '' && !/^[\s:=]/.test(after)) return undefined + return after +} + +function parseAttempt(after: string): string | undefined { + let rest = after.trimStart() + if (rest.startsWith(':') || rest.startsWith('=')) rest = rest.slice(1) + const tokens = rest.split(/\s+/).filter((t) => t !== '') + if (tokens.length !== 1) return undefined + const role = tokens[0].endsWith(';') ? tokens[0].slice(0, -1) : tokens[0] + return isDatatableRoleName(role) ? role : undefined +} + +export function parseMigrationRole(sql: string): MigrationRole { + for (const raw of sql.split('\n')) { + const line = raw.trim() + if (line === '') continue + if (!line.startsWith('--')) break + const after = roleAttempt(line) + if (after === undefined) continue + const role = parseAttempt(after) + return role === undefined ? { kind: 'malformed', line } : { kind: 'role', role } + } + return { kind: 'none' } +} + +/** + * `sql` declaring `role`: any role annotation attempt in the leading comment block is removed, + * and `-- role ` is prepended above everything, or nothing when `role` is undefined. + */ +export function withMigrationRole(sql: string, role: string | undefined): string { + if (role !== undefined && !isDatatableRoleName(role)) { + throw new Error(`Invalid data table role '${role}'`) + } + const lines = sql.split('\n') + const kept: string[] = [] + let i = 0 + for (; i < lines.length; i++) { + const line = lines[i].trim() + if (line !== '' && !line.startsWith('--')) break + if (roleAttempt(line) === undefined) kept.push(lines[i]) + } + const rest = [...kept, ...lines.slice(i)] + while (rest.length > 0 && rest[0].trim() === '') rest.shift() + return role === undefined ? rest.join('\n') : [`-- role ${role}`, ...rest].join('\n') +} diff --git a/frontend/src/lib/components/datatableUsableRoles.test.ts b/frontend/src/lib/components/datatableUsableRoles.test.ts new file mode 100644 index 0000000000..cca4ab7607 --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from 'vitest' + +const request = vi.fn() +vi.mock('$lib/gen/core/request', () => ({ request: (...args: unknown[]) => request(...args) })) +vi.mock('$lib/gen', () => ({ OpenAPI: {} })) +vi.mock('$lib/cloud', () => ({ isCloudHosted: () => false })) + +import { listUsableDatatableRoles } from './datatableUsableRoles' + +const refuse = (body: string) => () => + Promise.reject(Object.assign(new Error('Bad Request'), { body })) + +describe('listUsableDatatableRoles', () => { + // The refusal is recognised by its sentence, so rewording it on one side alone would turn every + // data table on a build without the Enterprise Edition into a failed lookup rather than one + // that is not under roles. + it('reads the enterprise refusal as not under roles, and rethrows anything else', async () => { + request.mockImplementation(refuse('Data table roles are a Windmill Enterprise Edition feature')) + expect(await listUsableDatatableRoles('ws', 'main')).toEqual({ + permissioned: false, + roles: [], + default_role: 'admin' + }) + + request.mockImplementation(refuse('Data table not found')) + const rethrown = await listUsableDatatableRoles('ws', 'main').catch((e) => e) + expect(rethrown).toBeInstanceOf(Error) + }) +}) diff --git a/frontend/src/lib/components/datatableUsableRoles.ts b/frontend/src/lib/components/datatableUsableRoles.ts new file mode 100644 index 0000000000..84e52f29a8 --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.ts @@ -0,0 +1,44 @@ +import { OpenAPI, type ListUsableDatatableRolesResponse } from '$lib/gen' +import { request } from '$lib/gen/core/request' +import { isCloudHosted } from '$lib/cloud' +import { ADMIN_DATATABLE_ROLE } from './dbTypes' + +// `datatable_roles_unavailable` on the server, which is a plain 400: rewording it there without +// here makes every role picker on a non-Enterprise build fail instead of reading "not under roles". +const ROLES_UNAVAILABLE = 'Data table roles are a Windmill Enterprise Edition feature' + +const NOT_UNDER_ROLES: ListUsableDatatableRolesResponse = { + permissioned: false, + roles: [], + default_role: ADMIN_DATATABLE_ROLE +} + +/** + * The roles the caller may connect as on a data table. Cloud has no instance database, so no data + * table there is under roles, and none of the role pickers show. Without the Enterprise Edition + * every roles route refuses, which reads the same way: the data table is then used the way it was + * before roles, and one that is under roles is refused when something connects to it. + */ +export async function listUsableDatatableRoles( + workspace: string, + datatableName: string +): Promise { + if (isCloudHosted()) return NOT_UNDER_ROLES + try { + // The generated client encodes path params with `encodeURI`, which leaves a '?' in a + // data table name created before names were restricted to cut the path short. + return await request( + { ...OpenAPI, ENCODE_PATH: encodeURIComponent }, + { + method: 'GET', + url: '/w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}', + path: { workspace, datatable_name: datatableName } + } + ) + } catch (e) { + const body = (e as { body?: unknown })?.body + const detail = `${typeof body === 'string' ? body : JSON.stringify(body ?? '')} ${(e as Error)?.message ?? e}` + if (detail.includes(ROLES_UNAVAILABLE)) return NOT_UNDER_ROLES + throw e + } +} diff --git a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts index 59345c6ae7..890635dcf8 100644 --- a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts +++ b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts @@ -5,7 +5,7 @@ import { isDbType } from './dbTypes' /** * Single URL param `dbm` encodes the full DB manager state: - * firstSegment~path~schema.table + * firstSegment~path~schema.table~role=name * * firstSegment: * datatable – database with datatable:// resource (resourceType always postgresql) @@ -26,28 +26,46 @@ import { isDbType } from './dbTypes' * datatable~main~.customers (schema "public" implied) * ducklake~main~.orders (schema "main" implied) * postgresql~$res:u/user/my_pg~public.customers + * datatable~main~.customers~role=analyst + * datatable~main~role=analyst (no schema/table selected) + * + * role=name (last segment, optional, data tables only): the data table role to connect as. + * Omitted means the data table's default role. A trailing segment starting with `role=` is always + * the role, whatever follows. The name is kept as written, even when invalid (a `.` included), so + * the connection refuses it visibly instead of falling back to the default. */ const dbManagerSchema = z.object({ dbm: z.string().nullable() }) -interface ParsedDbm { +export interface ParsedDbm { type: 'database' | 'datatable' | 'ducklake' path: string resType?: string schema?: string table?: string + role?: string } -function parseDbm(raw: unknown): ParsedDbm | null { +const ROLE_SEGMENT_PREFIX = 'role=' + +function isRoleSegment(segment: string | undefined): segment is string { + return !!segment && segment.startsWith(ROLE_SEGMENT_PREFIX) +} + +export function parseDbm(raw: unknown): ParsedDbm | null { if (!raw || typeof raw !== 'string') return null const parts = raw.split('~') if (parts.length < 2 || !parts[1]) return null const firstSeg = parts[0] const path = parts[1] - const schemaTable = parts[2] ?? '' + const rest = parts.slice(2) + const role = isRoleSegment(rest.at(-1)) + ? rest.pop()!.slice(ROLE_SEGMENT_PREFIX.length) + : undefined + const schemaTable = rest[0] ?? '' let type: ParsedDbm['type'] let resType: string | undefined @@ -81,12 +99,12 @@ function parseDbm(raw: unknown): ParsedDbm | null { schema = defaultSchemas[type] } - return { type, path, resType, schema, table } + return { type, path, resType, schema, table, role: type === 'datatable' ? role : undefined } } const defaultSchemas: Record = { datatable: 'public', ducklake: 'main' } -function buildDbm(p: ParsedDbm): string { +export function buildDbm(p: ParsedDbm): string { const firstSeg = p.type === 'database' ? p.resType! : p.type const schema = p.schema === defaultSchemas[p.type] ? undefined : p.schema let schemaTable = '' @@ -97,7 +115,12 @@ function buildDbm(p: ParsedDbm): string { } else if (schema) { schemaTable = `${schema}.` } - return schemaTable ? `${firstSeg}~${p.path}~${schemaTable}` : `${firstSeg}~${p.path}` + const segments = [firstSeg, p.path] + if (schemaTable) segments.push(schemaTable) + if (p.type === 'datatable' && p.role !== undefined) { + segments.push(`${ROLE_SEGMENT_PREFIX}${p.role}`) + } + return segments.join('~') } export interface DbManagerUriState { @@ -105,6 +128,8 @@ export interface DbManagerUriState { readonly effectiveInput: DbInput | undefined readonly isDatatableInput: boolean selectedDatatable: string | undefined + /** The data table role the drawer connects as; undefined means its default. */ + selectedRole: string | undefined selectedSchema: string | undefined selectedTable: string | undefined readonly open: boolean @@ -137,6 +162,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: 'database' as const, resourceType: resType as DbType, resourcePath: parsed.type === 'datatable' ? `datatable://${parsed.path}` : parsed.path, + role: parsed.role, specificSchema: parsed.schema, specificTable: parsed.table } @@ -163,6 +189,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: isDatatable ? 'datatable' : 'database', path: isDatatable ? nInput.resourcePath.slice('datatable://'.length) : nInput.resourcePath, resType: isDatatable ? undefined : nInput.resourceType, + role: isDatatable ? nInput.role : undefined, schema: nInput.specificSchema, table: nInput.specificTable }) @@ -194,7 +221,14 @@ export function useDbManagerUriState(): DbManagerUriState { return parsed?.type === 'datatable' ? parsed.path : undefined }, set selectedDatatable(v: string | undefined) { - if (v) updateField({ path: v }) + // A role belongs to one data table, so it cannot carry over to another. + if (v) updateField({ path: v, role: undefined }) + }, + get selectedRole() { + return parsed?.role + }, + set selectedRole(v: string | undefined) { + updateField({ role: v }) }, get selectedSchema() { return parsed?.schema diff --git a/frontend/src/lib/components/dbManagerRole.test.ts b/frontend/src/lib/components/dbManagerRole.test.ts new file mode 100644 index 0000000000..6c45d0ed30 --- /dev/null +++ b/frontend/src/lib/components/dbManagerRole.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { buildDbm, parseDbm } from './dbManagerDrawerModel.svelte' +import { schemaCacheKey } from './dbSchemaCache' +import { datatableReference, type DbInput } from './dbTypes' + +describe('dbm role segment', () => { + it('round-trips a role, with and without a table', () => { + for (const dbm of ['datatable~main~.orders~role=p4_analytics', 'datatable~main~role=p4-op']) { + expect(buildDbm(parseDbm(dbm)!)).toBe(dbm) + } + expect(parseDbm('datatable~main~sales.orders~role=analyst')).toMatchObject({ + path: 'main', + schema: 'sales', + table: 'orders', + role: 'analyst' + }) + }) + + it('reads a link without a role as the default role', () => { + const parsed = parseDbm('datatable~main~.orders')! + expect(parsed.role).toBeUndefined() + expect(parsed).toMatchObject({ schema: 'public', table: 'orders' }) + expect(buildDbm(parsed)).toBe('datatable~main~.orders') + }) + + it('keeps an invalid role as written, so the connection refuses it', () => { + expect(parseDbm('datatable~main~role=a;b')?.role).toBe('a;b') + // A dot does not turn it into a schema.table selection read as the default role. + expect(parseDbm('datatable~main~role=bad.name')).toMatchObject({ + role: 'bad.name', + schema: undefined, + table: undefined + }) + }) +}) + +describe('connecting as a role', () => { + const input = (role?: string): DbInput => ({ + type: 'database', + resourceType: 'postgresql', + resourcePath: 'datatable://main', + role + }) + + // What `getDatabaseArg` builds every DB manager connection from. + it('appends the role to the data table reference', () => { + expect(datatableReference('main', 'p4_analytics')).toBe('datatable://main?role=p4_analytics') + expect(datatableReference('main', undefined)).toBe('datatable://main') + }) + + it('never appends a role to a name containing ?', () => { + // The server reads such a whole reference as the stored name first, so `?role=` would + // be taken as part of the name or refused instead of picking the role. + expect(datatableReference('legacy?x', undefined)).toBe('datatable://legacy?x') + expect(() => datatableReference('sales?role=analytics', 'admin')).toThrow(/'\?' in its name/) + }) + + it('refuses a role name the server would not accept', () => { + expect(() => datatableReference('main', 'a&role=admin')).toThrow(/Invalid data table role/) + expect(() => datatableReference('main', '')).toThrow(/Invalid data table role/) + }) + + it('keys the schema cache by role', () => { + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input('b'))) + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input())) + }) +}) diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index a89abf4374..e239a9e957 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -8,12 +8,14 @@ import { runScriptAndPollResult } from './jobs/utils' import { writingJobOptions } from './jobs/writingJob' import type { DBSchema, SQLSchema } from '$lib/stores' import { stringifySchema } from './copilot/lib' -import type { DbInput, DbType } from './dbTypes' +import { datatableReference, type DbInput, type DbType } from './dbTypes' +import { withMigrationRole } from './datatableMigrationRole' import { assert } from '$lib/utils' import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' import { buildTableEditorValues, + type TableEditorForeignKey, type TableEditorValues } from './apps/components/display/dbtable/tableEditor' import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable' @@ -69,7 +71,9 @@ export function dbTableOpsWithPreviewScripts({ }): IDbTableOps { const dbType = getDbType(input) const language = getLanguageByResourceType(dbType) - const dbArg = getDatabaseArg(input) + // Built per call: an invalid role throws there, as that operation's error, rather than while + // the manager renders. + const dbArg = () => getDatabaseArg(input) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined function makeMarker(op: string, payload: Record): string { @@ -90,7 +94,7 @@ export function dbTableOpsWithPreviewScripts({ }) const result = await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, quicksearch }, language, content, tag } + requestBody: { args: { ...dbArg(), quicksearch }, language, content, tag } }) const count = result?.[0].count as number return count @@ -105,7 +109,7 @@ export function dbTableOpsWithPreviewScripts({ }) let items = (await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, ...params }, language, content, tag } + requestBody: { args: { ...dbArg(), ...params }, language, content, tag } })) as unknown[] if (!items || !Array.isArray(items)) { throw 'items is not an array' @@ -122,7 +126,7 @@ export function dbTableOpsWithPreviewScripts({ { workspace, requestBody: { - args: { ...dbArg, value_to_update: newValue, ...values }, + args: { ...dbArg(), value_to_update: newValue, ...values }, language, content, tag @@ -134,14 +138,14 @@ export function dbTableOpsWithPreviewScripts({ onDelete: async ({ values }) => { const content = makeMarker('DELETE', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) }, onInsert: async ({ values }) => { const content = makeMarker('INSERT', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) } @@ -245,11 +249,16 @@ export type IDbSchemaOps = { previewAlterSql: (params: { values: AlterTableValues; schema?: string }) => Promise onCreateSchema: (params: { schema: string }) => Promise onDeleteSchema: (params: { schema: string }) => Promise + onRenameSchema: (params: { schema: string; newSchema: string }) => Promise onFetchTableEditorDefinition: (params: { table: string schema?: string colDefs: TableMetadata }) => Promise + onFetchForeignKeys: (params: { + table: string + schema?: string + }) => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -278,7 +287,8 @@ export function dbSchemaOpsWithPreviewScripts({ tag?: string }): IDbSchemaOps { const dbType = getDbType(input) - const dbArg = getDatabaseArg(input) + // Built per call, for the same reason as in the table ops above. + const dbArg = () => getDatabaseArg(input) const language = getLanguageByResourceType(dbType) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined @@ -288,6 +298,8 @@ export function dbSchemaOpsWithPreviewScripts({ input.type === 'database' && input.resourcePath.startsWith('datatable://') ? input.resourcePath.slice('datatable://'.length) : undefined + // A migration declaring no role runs as admin, whatever role the manager connects as. + const migrationRole = input.type === 'database' ? (input.role ?? input.migrationRole) : undefined function makeMarker(op: string, payload: Record): string { if (ducklake) payload.ducklake = ducklake @@ -354,7 +366,7 @@ export function dbSchemaOpsWithPreviewScripts({ : undefined if (!datatableName || !status?.enabled) { await runScriptAndPollResult( - { workspace, requestBody: { args: dbArg, content, language, tag } }, + { workspace, requestBody: { args: dbArg(), content, language, tag } }, writingJobOptions ) return @@ -368,12 +380,16 @@ export function dbSchemaOpsWithPreviewScripts({ throw new MigrationRunCancelled() } } - const codeUp = wrapMigration(await expandMarker(workspace, language, content)) + // Wrapped before annotating: the annotation must lead, above `BEGIN;`. + const codeUp = withMigrationRole( + wrapMigration(await expandMarker(workspace, language, content)), + migrationRole + ) // Down migrations are only generated for Postgres for now. let codeDown: string | undefined if (downContent && dbType === 'postgresql') { const downSql = (await expandMarker(workspace, language, downContent)).trim() - if (downSql) codeDown = wrapMigration(downSql) + if (downSql) codeDown = withMigrationRole(wrapMigration(downSql), migrationRole) } const created = await WorkspaceService.createDatatableMigration({ workspace, @@ -396,6 +412,48 @@ export function dbSchemaOpsWithPreviewScripts({ } } + /** Resolves to [] when the database has no foreign key introspection + * (BigQuery) or the query fails: callers treat foreign keys as optional. */ + async function fetchForeignKeys({ + table, + schema + }: { + table: string + schema?: string + }): Promise { + if (dbType === 'bigquery') return [] + try { + const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) + const fkResult = await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg(), content: fkContent, language, tag } + }) + + let rawForeignKeys: RawForeignKey[] + if (dbType === 'snowflake') { + rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) + } else { + rawForeignKeys = fkResult as RawForeignKey[] + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + rawForeignKeys = rawForeignKeys.map((fk) => { + const lowerFk: any = {} + Object.keys(fk).forEach((key) => { + lowerFk[key.toLowerCase()] = fk[key] + }) + return lowerFk + }) + } + } + + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + return transformForeignKeys(rawForeignKeys) + } + } catch (e) { + console.warn('Failed to fetch foreign keys:', e) + } + return [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -454,51 +512,23 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onRenameSchema: async ({ schema, newSchema }) => { + const content = makeMarker('RENAME_SCHEMA', { schema, new_schema: newSchema }) + const downContent = makeMarker('RENAME_SCHEMA', { schema: newSchema, new_schema: schema }) + await applyDdl(migrationName('rename_schema', schema), content, downContent) + }, + onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { - let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] = - [] + const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined - // Fetch foreign keys (not supported for BigQuery) - if (dbType !== 'bigquery') { - try { - const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) - const fkResult = await runScriptAndPollResult({ - workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } - }) - - let rawForeignKeys: RawForeignKey[] - if (dbType === 'snowflake') { - rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) - } else { - rawForeignKeys = fkResult as RawForeignKey[] - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - rawForeignKeys = rawForeignKeys.map((fk) => { - const lowerFk: any = {} - Object.keys(fk).forEach((key) => { - lowerFk[key.toLowerCase()] = fk[key] - }) - return lowerFk - }) - } - } - - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - foreignKeys = transformForeignKeys(rawForeignKeys) - } - } catch (e) { - console.warn('Failed to fetch foreign keys:', e) - } - } - // Fetch primary key constraint name (not supported for BigQuery/MySQL) if (dbType !== 'bigquery' && dbType !== 'mysql') { try { const pkContent = makeMarker('PRIMARY_KEY_CONSTRAINT', { table, schema }) const pkResult = (await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: pkContent, language, tag } + requestBody: { args: dbArg(), content: pkContent, language, tag } })) as { constraint_name?: string; CONSTRAINT_NAME?: string }[] if (pkResult && Array.isArray(pkResult) && pkResult.length > 0) { @@ -597,7 +627,9 @@ export function getDefaultDbTag(input: DbInput): string { export function getDatabaseArg(input: DbInput | undefined) { if (input?.type === 'database') { if (input.resourcePath.startsWith('datatable://')) { - return { database: input.resourcePath } + return { + database: datatableReference(input.resourcePath.slice('datatable://'.length), input.role) + } } else { return { database: '$res:' + input.resourcePath } } diff --git a/frontend/src/lib/components/dbSchemaCache.ts b/frontend/src/lib/components/dbSchemaCache.ts new file mode 100644 index 0000000000..0421c0c033 --- /dev/null +++ b/frontend/src/lib/components/dbSchemaCache.ts @@ -0,0 +1,21 @@ +import type { DbInput } from './dbTypes' + +/** What identifies a database's schema, role included: two roles on one data table may reach + * different schemas, so they cannot share a cache entry. Never throws, since it keys derived + * state; the connection itself is what refuses an invalid role. */ +export function getDbSchemasPath(input: DbInput): string { + switch (input.type) { + case 'database': + return input.role !== undefined && input.resourcePath.startsWith('datatable://') + ? `${input.resourcePath}?role=${input.role}` + : input.resourcePath + case 'ducklake': + return 'ducklake://' + input.ducklake + } +} + +/** Scoped by the acting workspace: a data table of the same name can exist in both the nav and + * the acting workspace, and one's schema must not be reused for the other. */ +export function schemaCacheKey(workspace: string | undefined, input: DbInput): string { + return `${workspace}:${getDbSchemasPath(input)}` +} diff --git a/frontend/src/lib/components/dbTypes.ts b/frontend/src/lib/components/dbTypes.ts index 6a85617110..e6d3af12e8 100644 --- a/frontend/src/lib/components/dbTypes.ts +++ b/frontend/src/lib/components/dbTypes.ts @@ -3,6 +3,12 @@ export type DbInput = type: 'database' resourceType: DbType resourcePath: string + /** The data table role to connect as; the data table's default when unset. Only + * meaningful for a `datatable://` path. */ + role?: string + /** The role migrations written through this input declare when `role` is unset. A + * migration declaring none runs as admin, not as the role the manager connects as. */ + migrationRole?: string specificSchema?: string specificTable?: string } @@ -23,3 +29,47 @@ export const dbTypes = [ 'duckdb' ] as const export const isDbType = (str?: string): str is DbType => !!str && dbTypes.includes(str as DbType) + +/** The role every data table has: the one it connects as when it is not under roles. */ +export const ADMIN_DATATABLE_ROLE = 'admin' + +/** What the server accepts in `-- role ` and `?role=`. */ +export function isDatatableRoleName(name: string): boolean { + return /^[A-Za-z0-9_-]{1,63}$/.test(name) +} + +/** Whether a role can be named in a reference to this data table. A name stored before names were + * restricted may contain `?`, and the server reads such a whole reference as that name first, so + * `?role=` after it would be taken as part of the name or refused. */ +export function datatableNameTakesRole(name: string): boolean { + return !name.includes('?') +} + +/** The `migrationRole` of a data table that cannot name a role in its reference: it connects as + * its default role, which its migrations must then declare. */ +export function defaultMigrationRole( + name: string, + permissioned: boolean | undefined, + defaultRole: string | undefined +): string | undefined { + return permissioned && !datatableNameTakesRole(name) ? defaultRole : undefined +} + +/** `datatable://`, with `?role=` when a role is named. Throws rather than build a + * reference the executor would refuse, or one that would silently mean another role. */ +export function datatableReference(name: string, role: string | undefined): string { + if (role === undefined) return `datatable://${name}` + if (!isDatatableRoleName(role)) { + throw new Error( + `Invalid data table role '${role}': only letters, digits, '_' and '-' are allowed` + ) + } + if (!datatableNameTakesRole(name)) { + throw new Error( + `Data table '${name}' has a '?' in its name, so it can only be used here as its default role. Rename it to connect as role '${role}'.` + ) + } + return `datatable://${name}?role=${role}` +} + +export type DatatableRowAction = 'migrations' | 'roles' | 'export' | 'import' diff --git a/frontend/src/lib/components/dbt/DbtEditor.svelte b/frontend/src/lib/components/dbt/DbtEditor.svelte index f12e510c61..51a029c05a 100644 --- a/frontend/src/lib/components/dbt/DbtEditor.svelte +++ b/frontend/src/lib/components/dbt/DbtEditor.svelte @@ -8,10 +8,9 @@ // single file, so the arguments, the run and the graph are all the project's // whichever file happens to be open. import { untrack } from 'svelte' - import { createEventDispatcher, onDestroy } from 'svelte' + import { createEventDispatcher, onDestroy, onMount } from 'svelte' import type { Schema, SupportedLanguage } from '$lib/common' import type { Preview, ScriptModule } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { emptySchema } from '$lib/utils' import { inferArgs } from '$lib/infer' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -31,6 +30,15 @@ AssetGraphNodeData, DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import { + useDbtColumnLineage, + type DbtGraphPin + } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + EMPTY_COLUMN_GRAPH, + mergeColumnGraphs, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { DBT_DESCRIPTOR, DBT_MODULE_EXTENSIONS, @@ -45,6 +53,9 @@ import { ChevronDown, CornerDownLeft, Play, Plus } from 'lucide-svelte' import type { ScriptEditorWhitelabelCustomUi } from '../custom_ui' import { processSecretArgs } from '../secretArgUtils' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { schema = $bindable(), @@ -77,7 +88,7 @@ } = $props() const dispatch = createEventDispatcher() - let opWs = $derived(workspaceOverride ?? $workspaceStore) + let opWs = $derived(workspaceOverride ?? $operatingWorkspace) /** The open file, or `null` for the descriptor at the project root. */ let openFile = $state(null) @@ -97,6 +108,17 @@ } }) + // The stored schema can predate the parser (a CLI push, an older version), and + // the autosave baseline is taken as the mounted editor holds it + // (`schemaAsEditorMounts`), so the descriptor is re-inferred on mount rather + // than only on its first edit — otherwise the two only agree once edited. + onMount(async () => { + await inferSchema(code) + // Same single retry as ScriptEditor: the first parse can lose a transient + // wasm init race, and the baseline (`schemaAsEditorMounts`) retries too. + if (!validDescriptor && code) await inferSchema(code) + }) + function flushOpenFile() { if (openFile !== null && modules?.[openFile]) { modules[openFile] = { ...modules[openFile], content: editorCode } @@ -212,6 +234,28 @@ // the deployed graph, which previews by version instead. Either way the rows // come from the project whose SQL is displayed above them. let selectedBuffer = $state(undefined) + // Which graph the selection came from, so the lineage fetched below is the + // selected node's own project rather than whatever is deployed. + let selectionPin = $state(undefined) + // The selected model's column lineage, fetched on selection. Its own request + // rather than a field on the graph: only a project that opted into the + // analysis pass has any, and it is drawn for one model at a time. + const columnLineage = useDbtColumnLineage({ + workspace: () => opWs, + assetPaths: () => { + const path = selectedDbt ? selectedAsset?.path : undefined + return path ? [path] : [] + }, + pin: () => selectionPin + }) + // What the scripts around this project declare about its columns, off the + // same graph response the canvas drew. Merged rather than chosen between: a + // model's column and the ducklake column a script derives from it are one + // chain, and the trace has to cross that boundary. + let selectionProducerColumns = $state(EMPTY_COLUMN_GRAPH) + let selectionColumnGraph = $derived( + mergeColumnGraphs(columnLineage.graph, selectionProducerColumns) + ) let jobLoader: JobLoader | undefined = $state(undefined) let testJob: any = $state(undefined) @@ -514,10 +558,12 @@ testRunning={testIsLoading} testResult={testJob?.result} selection={graphSelection} - onSelect={(sel, dbt, buffer) => { + onSelect={(sel, dbt, buffer, pin, producerColumns) => { graphSelection = sel selectedDbt = dbt selectedBuffer = buffer + selectionPin = pin + selectionProducerColumns = producerColumns }} /> @@ -539,6 +585,10 @@ {args} fileInBundle={!!selectedDbt.original_file_path && !!modules?.[selectedDbt.original_file_path]} + columnGraph={selectionColumnGraph} + columnLoading={columnLineage.loading} + columnTruncated={columnLineage.truncated} + columnFailed={columnLineage.failed} onOpenFile={open} onClose={() => (graphSelection = undefined)} /> diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte index 2d25dbdc00..5863d00a59 100644 --- a/frontend/src/lib/components/dbt/DbtModelDetails.svelte +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -12,6 +12,9 @@ import { ClipboardCopy, Code2, FileCode2, Loader2, TableProperties, X } from 'lucide-svelte' import { copyToClipboard } from '$lib/utils' import type { DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import ColumnTraceSection from '$lib/components/assets/AssetGraph/ColumnTraceSection.svelte' + import DbtColumnList from '$lib/components/assets/AssetGraph/DbtColumnList.svelte' + import type { ColumnLineageGraph } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { previewDbtRows, type DbtPreview, type DbtPreviewBuffer } from './previewRows' import { nodeSelector } from './parseDbtRun' @@ -34,6 +37,13 @@ args, /** Whether this model's file is in the bundle being edited. */ fileInBundle = false, + /** The project's column-level lineage, when the descriptor asked for it. + * Fetched for this relation against the same graph the canvas draws, so + * the trace and the nodes above it describe one parse. */ + columnGraph, + columnLoading = false, + columnTruncated = false, + columnFailed = false, onOpenFile, onClose }: { @@ -45,6 +55,10 @@ buffer?: DbtPreviewBuffer args?: Record fileInBundle?: boolean + columnGraph?: ColumnLineageGraph + columnLoading?: boolean + columnTruncated?: boolean + columnFailed?: boolean onOpenFile?: (path: string) => void onClose?: () => void } = $props() @@ -111,7 +125,9 @@ return typeof v === 'object' ? JSON.stringify(v) : String(v) } - let columns = $derived(Object.entries(dbt.columns ?? {})) + let hasColumns = $derived( + !!dbt.column_schema?.length || Object.keys(dbt.columns ?? {}).length > 0 + ) // `dbt show` SELECTs from the node's own relation and the worker intersects // the selector with `resource_type:model`, so offering it on a seed, snapshot // or source only ever produces a failed job. @@ -202,15 +218,15 @@ {#if stalePlaceholders}
    - The run arguments have changed since this graph was parsed, so these rows need not - describe the models on screen — arguments reach schemas, aliases and which models exist - at all. Refresh the models to draw and preview them under the current ones. + The run arguments have changed since this graph was parsed, so these rows need not describe + the models on screen — arguments reach schemas, aliases and which models exist at all. Refresh + the models to draw and preview them under the current ones.
    {:else if staleVars}
    - The run form's vars have changed since this graph was parsed. Rows are previewed under - the vars it was parsed with, so they still describe the models on screen — refresh the - models to draw and preview them under the current ones. + The run form's vars have changed since this graph was parsed. Rows are previewed under the + vars it was parsed with, so they still describe the models on screen — refresh the models to + draw and preview them under the current ones.
    {/if} @@ -234,26 +250,9 @@ {/if} - {#if columns.length > 0 || (dbt.data_tests?.length ?? 0) > 0} + {#if hasColumns || (dbt.data_tests?.length ?? 0) > 0}
    - {#if columns.length > 0} -
    -
    columns declared
    -
    - {#each columns as [name, desc] (name)} -
    - {name} - {desc} -
    - {/each} -
    - -
    - Declared metadata — dbt reports no column-level lineage. -
    -
    - {/if} + {#if (dbt.data_tests?.length ?? 0) > 0}
    tests
    @@ -269,6 +268,16 @@
    {/if} + + {#if showRows && preview} {#if 'error' in preview}
    {preview.error}
    diff --git a/frontend/src/lib/components/dbt/DbtModelGraph.svelte b/frontend/src/lib/components/dbt/DbtModelGraph.svelte index e6d948d926..bd7f42a9e7 100644 --- a/frontend/src/lib/components/dbt/DbtModelGraph.svelte +++ b/frontend/src/lib/components/dbt/DbtModelGraph.svelte @@ -25,6 +25,12 @@ DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' import { useDbtRunStatus } from './runStatus.svelte' + import type { DbtGraphPin } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + buildColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' let { workspace, @@ -80,7 +86,19 @@ * buffer rather than a deployed version — as submitted, not as the * editor holds it now. Sent with the selection rather than exposed on * its own so it can never disagree with the SQL the parent shows. */ - buffer: DbtPreviewBuffer | undefined + buffer: DbtPreviewBuffer | undefined, + /** Which graph this node was taken from, so anything else fetched + * about it describes the same project: the editor's own parse job + * when the panel is pinned to one, else the deployed version. Sent + * with the selection for the same reason the buffer is — it must not + * be able to disagree with the node on screen. */ + pin: DbtGraphPin, + /** Column lineage the CONSUMERS of this project declare — a script + * reading a model's column and writing a ducklake one. It comes off + * the same graph response, and the details pane merges it with the + * project's own so a trace crosses that boundary instead of ending + * at it. */ + producerColumns: ColumnLineageGraph ) => void } = $props() @@ -364,6 +382,25 @@ // graph that actually came back. let editorParsed = $derived(refreshJob != undefined && raw?.dbt_snapshot_job === refreshJob) + // Which stored graph is on screen. Anything the details pane fetches about a + // selected node asks for this one, so it cannot describe a node parsed from + // the buffer with the deployed version's answer. + let pin = $derived( + editorParsed && refreshJob ? { jobId: refreshJob } : { scriptHash: deployedHash } + ) + + // What the scripts around this project declare about its columns. Empty for + // the ordinary project nothing downstream annotates. + // + // A consumer is anchored here only by its `// materialize` target: this graph + // is fetched `asset_kinds=dbt` so the canvas is the project and nothing else, + // and `buildColumnGraph`'s other anchor is a ducklake WRITE EDGE, which that + // filter drops. So a script consuming a model and writing a ducklake table it + // never declared contributes no hop in this editor, while it does on the + // pipeline page, whose graph spans both kinds. Widening the request would put + // ducklake nodes on the dbt canvas, which is the opposite of what it is for. + let producerColumns = $derived(graph ? buildColumnGraph(graph) : EMPTY_COLUMN_GRAPH) + // `untrack`, because the effect that reloads the graph clears the selection // through here: reading the graph to describe a selection would subscribe that // effect to the very state its own fetch writes, and it would reload forever. @@ -374,7 +411,9 @@ sel?.kind === 'asset' ? graph?.assets.find((a) => a.kind === sel.asset_kind && a.path === sel.path)?.dbt : undefined, - editorParsed ? parsedBuffer : undefined + editorParsed ? parsedBuffer : undefined, + pin, + producerColumns ) ) } @@ -405,7 +444,6 @@ if (deployedHash != undefined) return 'as of last deploy' return 'never parsed' }) -
    @@ -435,8 +473,8 @@ {#if refreshPending}
    - Still parsing. A cold worker provisions the dbt engine before it starts; a project - pinned to a worker tag nothing serves waits here indefinitely. + Still parsing. A cold worker provisions the dbt engine before it starts; a project pinned to a + worker tag nothing serves waits here indefinitely. !b.narrow)) + + function dropdownHost(btn: MainButton): MainButton | undefined { + if (typeof btn.narrow !== 'object') return undefined + const label = btn.narrow.dropdownOf + return barButtons.find((b) => b.label === label && !b.buttonProps.disabled) + } + + async function toggleErrorHandler() { + const next = await toggleWorkspaceErrorHandler( + errorHandlerKind, + scriptOrFlowPath, + errorHandlerMuted + ) + if (next !== undefined) errorHandlerMuted = next + } + + const allMenuItems: Item[] = $derived([ + ...(wide.current ? [] : mainButtons.filter((b) => b.narrow && !dropdownHost(b))).map((b) => ({ + displayName: b.label, + description: b.description, + icon: b.buttonProps.startIcon, + href: b.buttonProps.href, + action: b.buttonProps.onClick, + disabled: b.buttonProps.disabled, + type: 'action' as const + })), + ...(wide.current + ? [] + : [ + { + displayName: errorHandlerMuted ? 'Unmute error handler' : 'Mute error handler', + icon: errorHandlerMuted ? BellOff : Bell, + action: toggleErrorHandler, + type: 'action' as const + } + ]), + ...menuItems.map((item, i) => ({ + displayName: item.label, + icon: item.Icon, + action: item.onclick, + type: item.color === 'red' ? ('delete' as const) : ('action' as const), + separatorTop: i === 0 && !wide.current + })) + ]) + + function dropdownItemsOf(host: MainButton) { + if (wide.current) return undefined + const items = mainButtons + .filter((b) => dropdownHost(b) === host) + .map((b) => ({ + label: b.label, + description: b.description, + icon: b.buttonProps.startIcon, + href: b.buttonProps.href, + onClick: b.buttonProps.onClick, + disabled: b.buttonProps.disabled + })) + return items.length > 0 ? items : undefined + }
    @@ -103,38 +176,26 @@ {@render trigger_badges?.()}
    - {#if menuItems.length > 0} - {#key menuItems} - ({ - displayName: item.label, - icon: item.Icon, - action: item.onclick, - type: item.color === 'red' ? 'delete' : 'action' - }))} - placement="bottom-end" - size="md" - /> + {#if allMenuItems.length > 0} + {#key allMenuItems} + {/key} {/if} - - {#each mainButtons as btn} + {#if wide.current} + + {/if} + {#each barButtons as btn (btn.label)} + {@const dropdownItems = dropdownItemsOf(btn)} - diff --git a/frontend/src/lib/components/details/DetailPageLayout.svelte b/frontend/src/lib/components/details/DetailPageLayout.svelte index be6e1a9e79..a6a2aa7a8a 100644 --- a/frontend/src/lib/components/details/DetailPageLayout.svelte +++ b/frontend/src/lib/components/details/DetailPageLayout.svelte @@ -1,5 +1,6 @@
    @@ -54,7 +74,7 @@
    - {@render form?.()} + {@render form?.({ graphInline: true })} @@ -65,7 +85,11 @@ {@render save_inputs_render?.()} {/snippet} {#snippet flow_step()} - {@render flow_step_render?.()} + +
    + {@render flow_step_render?.({})} +
    {/snippet} {#snippet triggers()} {@render triggers_render?.()} @@ -79,12 +103,15 @@
    {@render header?.()}
    - + + {#if !isChatMode} {/if} - {#if isChatMode && flow_json} + {#if flow_json} {/if} {#if !isOperator} @@ -99,7 +126,7 @@ {#snippet content()}
    - {@render form?.()} + {@render form?.({ graphInline: false })} @@ -108,9 +135,9 @@ {@render triggers?.()} - {#if isChatMode && flow_json} + {#if flow_json} - {@render flow_graph_render?.()} + {@render pagedGraph()} {/if} {#if flow_json} @@ -128,3 +155,34 @@
    {/if}
    + + +{#snippet pagedGraph()} + { + if (key === 'graph') selected = 'saved_inputs' + }} + pages={[ + { key: 'graph', content: graphPageContent }, + { key: 'step', content: stepPageContent } + ]} + /> +{/snippet} + +{#snippet graphPageContent()} +
    + {@render flow_graph_render?.()} +
    +{/snippet} + +{#snippet stepPageContent()} + +
    + {@render flow_step_render?.({ onBack: () => (selected = 'saved_inputs') })} +
    +{/snippet} diff --git a/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte b/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte index 9c0b964c9f..6097a235ca 100644 --- a/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte +++ b/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte @@ -2,58 +2,21 @@ import { Bell, BellOff } from 'lucide-svelte' import { Button } from '$lib/components/common' - import { FlowService, ScriptService } from '$lib/gen' - import { sendUserToast } from '$lib/toast' - import { workspaceStore } from '$lib/stores' import Tooltip from '../Tooltip.svelte' + import { toggleWorkspaceErrorHandler } from './errorHandlerToggle' interface Props { - kind: 'script' | 'flow'; - scriptOrFlowPath: string; - errorHandlerMuted: boolean | undefined; - iconOnly?: boolean; + kind: 'script' | 'flow' + scriptOrFlowPath: string + errorHandlerMuted: boolean | undefined + iconOnly?: boolean } - let { - kind, - scriptOrFlowPath, - errorHandlerMuted = $bindable(), - iconOnly = true - }: Props = $props(); + let { kind, scriptOrFlowPath, errorHandlerMuted = $bindable(), iconOnly = true }: Props = $props() async function toggleErrorHandler(): Promise { - if ($workspaceStore !== undefined) { - try { - if (kind === 'flow') { - await FlowService.toggleWorkspaceErrorHandlerForFlow({ - workspace: $workspaceStore, - path: scriptOrFlowPath, - requestBody: { - muted: !errorHandlerMuted - } - }) - } else { - await ScriptService.toggleWorkspaceErrorHandlerForScript({ - workspace: $workspaceStore, - path: scriptOrFlowPath, - requestBody: { - muted: !errorHandlerMuted - } - }) - } - } catch (error) { - sendUserToast( - `Error while toggling Workspace Error Handler: ${error.body || error.message}`, - true - ) - return - } - errorHandlerMuted = !errorHandlerMuted - sendUserToast( - errorHandlerMuted ? 'Workspace error handler muted' : 'Workspace error handler active', - false - ) - } + const next = await toggleWorkspaceErrorHandler(kind, scriptOrFlowPath, errorHandlerMuted) + if (next !== undefined) errorHandlerMuted = next } diff --git a/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte b/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte index 51cb389616..2aec483e32 100644 --- a/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte +++ b/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte @@ -3,7 +3,9 @@ import { FlowService, ScriptService } from '$lib/gen' import { sendUserToast } from '$lib/toast' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { kind: 'script' | 'flow' @@ -25,11 +27,11 @@ async function toggleErrorHandler(): Promise { toggleState = !toggleState - if ($workspaceStore !== undefined) { + if ($operatingWorkspace !== undefined) { try { if (kind === 'flow') { await FlowService.toggleWorkspaceErrorHandlerForFlow({ - workspace: $workspaceStore, + workspace: $operatingWorkspace, path: scriptOrFlowPath, requestBody: { muted: !errorHandlerMuted @@ -37,7 +39,7 @@ }) } else { await ScriptService.toggleWorkspaceErrorHandlerForScript({ - workspace: $workspaceStore, + workspace: $operatingWorkspace, path: scriptOrFlowPath, requestBody: { muted: !errorHandlerMuted diff --git a/frontend/src/lib/components/details/errorHandlerToggle.ts b/frontend/src/lib/components/details/errorHandlerToggle.ts new file mode 100644 index 0000000000..1a7ab5296a --- /dev/null +++ b/frontend/src/lib/components/details/errorHandlerToggle.ts @@ -0,0 +1,39 @@ +import { get } from 'svelte/store' +import { FlowService, ScriptService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { workspaceStore } from '$lib/stores' + +/** Flips the workspace error handler for one script or flow and reports the outcome in a + * toast. Returns the new muted state, or undefined when nothing changed. */ +export async function toggleWorkspaceErrorHandler( + kind: 'script' | 'flow', + path: string, + muted: boolean | undefined +): Promise { + const workspace = get(workspaceStore) + if (workspace === undefined) return undefined + const next = !muted + try { + if (kind === 'flow') { + await FlowService.toggleWorkspaceErrorHandlerForFlow({ + workspace, + path, + requestBody: { muted: next } + }) + } else { + await ScriptService.toggleWorkspaceErrorHandlerForScript({ + workspace, + path, + requestBody: { muted: next } + }) + } + } catch (error) { + sendUserToast( + `Error while toggling Workspace Error Handler: ${error.body || error.message}`, + true + ) + return undefined + } + sendUserToast(next ? 'Workspace error handler muted' : 'Workspace error handler active', false) + return next +} diff --git a/frontend/src/lib/components/diff_drawer.ts b/frontend/src/lib/components/diff_drawer.ts index c6e9b5cdf8..c1e7e8980a 100644 --- a/frontend/src/lib/components/diff_drawer.ts +++ b/frontend/src/lib/components/diff_drawer.ts @@ -1,9 +1,53 @@ import type { Value } from '$lib/utils' +export type DiffVersionOption = { + /** Opaque to the drawer — a script hash, a flow version id, an app version id. */ + id: string + /** Identifies the version: its number and hash. Carries the weight in the list. */ + label: string + /** Who deployed it, and when — rendered under the label in secondary text, so the + * version reads first and the attribution second. */ + subtitle?: string + /** The version currently deployed. */ + isHead?: boolean +} + +/** How many deployed versions the diff picker asks for at a time. The history endpoints + * still answer whole when nobody asks — the panels and the CLI read them that way — so + * this is the picker's own appetite, not their default. */ +export const VERSION_PAGE_SIZE = 20 + export type DiffDrawerDiff = | { mode: 'normal' deployed: Value + /** Names the deployed side, e.g. `Deployed d2154d55 by bob`. The version + * identity is stripped from the displayed metadata, so without this the + * reader cannot tell what their draft is being compared against. */ + deployedLabel?: string + /** Deployed versions the reader can compare against, newest first. The + * drawer only renders the picker; each editor supplies its own list and + * fetcher because scripts, flows and apps identify versions differently. */ + versions?: DiffVersionOption[] + /** Loads one version's payload. Returning `undefined` leaves the current + * comparison in place rather than blanking the diff. */ + loadVersion?: (id: string) => Promise + /** Fetches the next, older page of `versions`. `versions` holds one page so the + * drawer opens without waiting on a path a pipeline has deployed thousands of + * times; the reader asks for the rest. Returning nothing ends the list. */ + loadMoreVersions?: () => Promise + /** Moves the draft's base to the head and keeps its content, rendered as a + * header action so the user takes the latest with the diff in front of them. + * Called with the version the drawer is showing as head (from `versions`), so + * the base adopted is the one the reader just looked at. */ + onTakeLatest?: (head?: string) => void | Promise + /** The version the draft forked from. The drawer offers `onTakeLatest` only + * while it differs from the head, and passes that head to it. */ + draftBase?: string + /** The version the deployed payload passed here came from, used as the head when + * no `versions` list loaded. Without either, the drawer cannot tell whether the + * draft is behind and offers nothing. */ + deployedHead?: string draft?: Value | undefined current: Value defaultDiffType?: 'deployed' | 'draft' @@ -18,7 +62,19 @@ export type DiffDrawerDiff = } export interface DiffDrawerI { - openDrawer: () => void + /** Pass the token from `beginOpening` to continue that opening; called without one, + * the drawer claims a fresh opening, so any reuse invalidates a fetch still in + * flight rather than being overwritten by it. */ + openDrawer: (token?: number) => void closeDrawer: () => void - setDiff: (diff: DiffDrawerDiff) => void + /** Takes the opening's token on the same terms as `openDrawer`. */ + setDiff: (diff: DiffDrawerDiff, token?: number) => void + /** Claim the drawer for one opening. Filling it takes awaited fetches, and a path + * change remounts the editor while this drawer stays, so the token lives here: an + * editor checks `ownsOpening` before every write and drops the opening it started + * when it goes away. */ + beginOpening: () => number + ownsOpening: (token: number) => boolean + /** Drop that opening and, when it still owns the drawer, what it put on screen. */ + abandonOpening: (token: number) => void } diff --git a/frontend/src/lib/components/displayNameLoaders.ts b/frontend/src/lib/components/displayNameLoaders.ts new file mode 100644 index 0000000000..5eed1e81a0 --- /dev/null +++ b/frontend/src/lib/components/displayNameLoaders.ts @@ -0,0 +1,71 @@ +import { get } from 'svelte/store' +import { IntegrationService, ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { addResourceTypeDisplayName, setHubIntegrationDisplayNames } from './resourceTypeDisplay' + +/** + * Loads what `resourceTypeDisplayName` and `integrationDisplayName` read: a type's stored name for a + * surface that holds no row for it, and the hub's integration list, which every picker that needs + * it shares. Apart from `resourceTypeDisplay`, which makes no API calls so it can be unit-tested + * alone. Cached briefly: drawers and pickers reopen often, and a name rarely changes. + */ +const CACHE_MS = 60_000 + +const resourceTypeRowCached = createCache( + ({ workspace, name }: { workspace: string; name: string }) => + ResourceService.getResourceType({ workspace, path: name }).then( + (rt) => addResourceTypeDisplayName(rt), + () => {} + ), + { invalidateMs: CACHE_MS, maxSize: 50 } +) + +/** + * Fill `resourceTypeDisplayName` for one type, for a surface titled with a type it holds no row + * for. The name is stored with the type, so this reads the row rather than the hub. + */ +export function loadResourceTypeDisplayName(workspace: string, name: string): Promise { + return resourceTypeRowCached({ workspace, name }) +} + +/** Bumped by every failed read, so the next caller keys a fresh read rather than the rejection. */ +let failedReads = 0 + +const hubIntegrationsCached = createCache( + ({ kind }: { kind?: string; refresh: number; attempt: number }) => + IntegrationService.listHubIntegrations({ kind }).then( + (integrations) => { + setHubIntegrationDisplayNames(integrations) + return integrations + }, + (error) => { + failedReads += 1 + throw error + } + ), + { invalidateMs: CACHE_MS } +) + +/** + * The hub's integration list, read once a minute per `kind` however many pickers ask, recording + * each integration's name on the way. A failed read rejects, so a picker can say the hub is + * unavailable, but is not kept: the next caller reads again. `refresh` is a picker's refresh + * count, and a new value reads again too. + */ +export function listHubIntegrationsShared(kind?: string, refresh = 0) { + return hubIntegrationsCached({ kind, refresh, attempt: failedReads }) +} + +/** + * Fill `integrationDisplayName` for a picker whose integrations come from its own items rather + * than the hub's integration list, as the hub app and flow pickers do. Unfiltered: `kind` + * narrows by script kind, so asking for an app or a flow would name nothing. + */ +export function loadHubIntegrationDisplayNames(): Promise { + if (get(disableHubStore)) return Promise.resolve() + return listHubIntegrationsShared().then( + () => {}, + () => {} + ) +} diff --git a/frontend/src/lib/components/editorStoragePathContext.ts b/frontend/src/lib/components/editorStoragePathContext.ts new file mode 100644 index 0000000000..e3685995c4 --- /dev/null +++ b/frontend/src/lib/components/editorStoragePathContext.ts @@ -0,0 +1,19 @@ +import { getContext, setContext } from 'svelte' + +// The path an editor's draft is stored under, the same key the live-editor draft +// registers. Published by whichever ancestor owns it (the session tab, or the +// full-page editor) because the parts that must say which item an editor is open +// on sit below both, and the path they can see themselves is the renamed one. + +const KEY = 'EditorStoragePath' + +/** `undefined` where the editor has no stored draft to speak of (a drawer mount). */ +export type EditorStoragePath = () => string | undefined + +export function setEditorStoragePath(storagePath: EditorStoragePath): void { + setContext(KEY, storagePath) +} + +export function getEditorStoragePath(): EditorStoragePath | undefined { + return getContext(KEY) +} diff --git a/frontend/src/lib/components/flow_builder.ts b/frontend/src/lib/components/flow_builder.ts index ce848c6f83..f763e92faa 100644 --- a/frontend/src/lib/components/flow_builder.ts +++ b/frontend/src/lib/components/flow_builder.ts @@ -10,6 +10,14 @@ import type { WorkspaceItem } from './workspacePicker' export type FlowBuilderProps = { initialPath?: string + /** The draft row's own path (the URL path). `initialPath` is re-seeded from the + * draft's `draft_path` so the topbar shows the pending name, so it can't be used + * to resolve what is actually deployed. */ + userDraftPath?: string + /** Moves the draft's base to the head and keeps its content; offered in the diff + * drawer while the draft is behind, and called with the version the drawer shows + * as head. */ + onTakeLatest?: (head?: string) => void | Promise pathStoreInit?: string | undefined newFlow: boolean selectedId: string | undefined @@ -24,9 +32,10 @@ export type FlowBuilderProps = { disabledFlowInputs?: boolean savedPrimarySchedule?: ScheduleTrigger | undefined // used to set the primary schedule in the legacy primaryScheduleStore version?: number | undefined - /** flow_version the draft was forked from; when set, the deploy-time staleness - * check compares it (not the load-time head `version`) against the latest. */ - draftBaseVersion?: number | undefined + /** flow_version the draft was forked from (as text, like `draft_base`); when + * set, the deploy-time staleness check compares it (not the load-time head + * `version`) against the latest. */ + draftBaseVersion?: string | undefined draftTriggersFromUrl?: Trigger[] | undefined selectedTriggerIndexFromUrl?: number | undefined children?: import('svelte').Snippet diff --git a/frontend/src/lib/components/flows/ChatFlowBadge.svelte b/frontend/src/lib/components/flows/ChatFlowBadge.svelte new file mode 100644 index 0000000000..1787528269 --- /dev/null +++ b/frontend/src/lib/components/flows/ChatFlowBadge.svelte @@ -0,0 +1,20 @@ + + + + Chat + diff --git a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte index 947eca1a67..86fff85437 100644 --- a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte +++ b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte @@ -43,11 +43,13 @@ type RawScript } from '$lib/gen' import { deepEqual } from 'fast-equals' - import { workspaceStore } from '$lib/stores' import S3FilePicker from '../S3FilePicker.svelte' import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte' import { watch } from 'runed' import { sendUserToast } from '$lib/toast' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { modules, @@ -63,7 +65,7 @@ const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') const { selectionManager, opWorkspace } = getContext('FlowEditorContext') || {} - let opWs = $derived(opWorkspace?.() ?? $workspaceStore) + let opWs = $derived(opWorkspace?.() ?? $operatingWorkspace) // Expose the acting workspace to the asset explore controls (ExploreAssetButton // reads it from this context; the DB manager / S3 picker act on it). $effect(() => { diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index eeb4101f92..a9c3a40e8a 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -2,6 +2,9 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import Disposable from '$lib/components/common/drawer/Disposable.svelte' import FlowEditorPanel from './content/FlowEditorPanel.svelte' + import { agentEditorTarget, type AgentEditorTarget } from './agentEditorStore.svelte' + import AgentEditorModal from './content/AgentEditorModal.svelte' + import { repointLinkedAgent } from './linkedAgentDrafts' import FlowModuleSchemaMap from './map/FlowModuleSchemaMap.svelte' import type { OpenInSessionSource } from '$lib/components/sessions/OpenInSessionButton.svelte' import WindmillIcon from '../icons/WindmillIcon.svelte' @@ -14,7 +17,7 @@ import { useFlowPanelMode } from './flowPanelMode.svelte' import { useFlowPanelPlacementTelemetry } from './flowEditorTelemetry' - import { writable } from 'svelte/store' + import { get, writable } from 'svelte/store' import type { PropPickerContext, FlowPropPickerConfig } from '$lib/components/prop_picker' import type { PickableProperties } from '$lib/components/flows/previousResults' import type { Flow, Job } from '$lib/gen' @@ -37,15 +40,29 @@ import FlowPanelPlacementPicker from './common/FlowPanelPlacementPicker.svelte' import { prefersSessionHandoff } from '../copilot/chat/global/gate' import { openSourceInSession } from '$lib/components/sessions/sessionSwitch.svelte' - import { userStore } from '$lib/stores' - const { flowStore, selectionManager } = getContext('FlowEditorContext') + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) + const { flowStore, selectionManager, pathStore, opWorkspace } = + getContext('FlowEditorContext') + // Flow paths repeat across workspaces, and a session keeps every tab it has visited alive, so two + // editors can hold the same path at once. Both halves are needed to tell them apart. + let editorWorkspace = $derived(opWorkspace?.() ?? $operatingWorkspace) + function targetWorkspace(t: AgentEditorTarget): string | undefined { + return t.workspace ?? $operatingWorkspace + } + const sessionScopedManager = getContext('aiChatManager') const aiChatManager = sessionScopedManager ?? singletonAiChatManager interface Props { loading: boolean disableStaticInputs?: boolean - disableTutorials?: boolean disableAi?: boolean disableSettings?: boolean disabledFlowInputs?: boolean @@ -89,7 +106,6 @@ let { loading, disableStaticInputs = false, - disableTutorials = false, disableAi = false, disableSettings = false, disabledFlowInputs = false, @@ -231,7 +247,19 @@ setContext('PropPickerContext', { flowPropPickerConfig, pickablePropertiesFiltered: writable(undefined), - inModalPanel: () => panelMode === 'modal' + // The agent editor is a dialog over the same graph, so a connect started inside it has the + // same closure hazard as one started from the modal panel. Only this flow's own, on the same + // rule the mount below claims one by: a session keeps every visited tab alive, and a target + // belonging to another of them is not a dialog over this graph. + inModalPanel: () => { + if (panelMode === 'modal') return true + const t = agentEditorTarget() + return ( + t !== undefined && + t.host?.flowPath === get(pathStore) && + targetWorkspace(t) === editorWorkspace + ) + } }) // Read by graph step items (VirtualItem) to show a per-step "explore" hint on hover, @@ -369,7 +397,6 @@ bind:this={flowModuleSchemaMap} controlsPosition={compactGraphOverlay ? 'bottom' : 'top'} {disableStaticInputs} - {disableTutorials} {disableAi} {disableSettings} {smallErrorHandler} @@ -384,7 +411,7 @@ if ( !sessionScopedManager && sessionOpen && - prefersSessionHandoff($userStore?.operator) + prefersSessionHandoff(actingUser?.operator) ) { void openSourceInSession(sessionOpen, { previewParams: { selected: detail.moduleId }, @@ -524,3 +551,13 @@ {/if} {/snippet} + + + t.host?.flowPath === $pathStore && targetWorkspace(t) === editorWorkspace} + onRenamed={(from, to) => repointLinkedAgent(flowStore.val.value, from, to)} +/> diff --git a/frontend/src/lib/components/flows/FlowEditorTutorial.svelte b/frontend/src/lib/components/flows/FlowEditorTutorial.svelte deleted file mode 100644 index ca1972914e..0000000000 --- a/frontend/src/lib/components/flows/FlowEditorTutorial.svelte +++ /dev/null @@ -1,62 +0,0 @@ - - -{#key $tutorialsToDo} - - {#snippet buttonReplacement()} - + {/if} + + {/if} +
    + {/snippet} +
    + +
    + draft?.deployed} + getCurrent={() => draft?.state} + onDiscard={() => draft?.sync.resetToDeployed(target?.path ?? '')} + title="Deployed <> Unsaved agent changes" + /> +
    + + +
    + + {/key} + + + {#snippet agentPage()} + showAgentEditorTool(id)} + {onSaved} + /> + {/snippet} + + {#snippet evalsPage()} + + {/snippet} + + + versionDrawer?.closeDrawer()} noPadding> + { + versionDrawer?.closeDrawer() + // A restore writes the resource as a deploy does, so the flow behind has to be told + // the same way. Captured before closing, which drops the target this reads. + const at = currentWriteTarget() + // Close the editor too, as the generic resource editor does on a restore: it holds + // a baseline captured before the restore, and any local draft on top of it, so + // deploying from it afterwards would write the pre-restore value straight back over + // the version just restored. + close() + if (at) reconcileQuietly(at, at.path) + }} + /> + + +{/if} diff --git a/frontend/src/lib/components/flows/content/AgentMemoryNotes.svelte b/frontend/src/lib/components/flows/content/AgentMemoryNotes.svelte new file mode 100644 index 0000000000..491a26daa3 --- /dev/null +++ b/frontend/src/lib/components/flows/content/AgentMemoryNotes.svelte @@ -0,0 +1,152 @@ + + +{#if on && !s3StorageConfigured} +

    + Without S3 storage on the workspace, memory is kept in the database, up to 100KB per memory. +

    +{/if} +{#if legacyMessages} + +
    + + An earlier version of the editor saved these previous messages inside the memory setting, + and this agent still sends them. + + {#if historyOnStep} +
    + +
    + {/if} +
    +
    +{/if} +{#if legacyMemoryId} + +
    + + {historyOnStep + ? 'Fixed memory id generated when this flow was saved.' + : 'Fixed memory id saved with this agent.'} + Every run shares it unless the caller passes one. + +
    + {#if historyOnStep} + + {/if} + +
    +
    +
    +{/if} +{#if legacyEquivalent} + +
    + + An earlier version of the editor saved this setting. It works the same as {on + ? 'On' + : 'Off'}. + +
    + +
    +
    +
    +{/if} diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 687dc9d98c..83be7c876e 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -1,22 +1,17 @@
    @@ -549,6 +582,15 @@ v{version} {/if} + {#if hasDraft} + + Draft + {#snippet text()} + This agent has unsaved changes. Testing this flow runs the draft, and deploying the + flow offers to deploy it. + {/snippet} + + {/if}
    {#if brainParams.length > 0 || inheritedTools.length > 0} @@ -560,40 +602,32 @@ {/if} {/if} - -
    {#if showDetail && (brainParams.length > 0 || inheritedTools.length > 0)} @@ -617,7 +651,32 @@ {/if}
    - {#if !providerOk} + {#if unavailable === 'forbidden'} +
    + + You don't have access to {agent}, so its configuration + can't be shown or edited here. + +
    + {:else if unavailable === 'missing'} +
    + + No saved agent exists at {agent}. It may have been + renamed or deleted. Remove the link to configure the step here, or add the agent again + from Saved agents. +
    + +
    +
    +
    + {:else if !providerOk}
    This agent's model provider{#if providerPath} @@ -626,73 +685,7 @@
    {/if} - {:else if editingPath} -
    -
    - -
    -
    - {editingPath} - {#if version != undefined} - - v{version} - - {/if} - {#if edited} - - unsaved changes - - {/if} -
    -
    - saving updates every flow using it - {#snippet text()} - The edits live in this step until you decide: Evals runs them as they are here, Save - changes writes them to the agent, Cancel drops them and re-links the step. - {/snippet} - -
    -
    - -
    -
    - - -
    -
    - {#if providerSaveError} -

    - {providerSaveError} -

    - {/if} - {:else} + {:else if !fromAgentEditor}
    @@ -740,7 +736,10 @@
    - {#if schemas[tool.id] !== undefined && localArgs[tool.id] !== undefined} + + {#if !open} + + {:else if schemas[tool.id] !== undefined && localArgs[tool.id] !== undefined} localArgs[tool.id], (v) => { @@ -163,7 +200,7 @@ {:else}
    Loading inputs...
    {/if} - {#if code} + {#if open && code}
    Tool code (read-only) diff --git a/frontend/src/lib/components/flows/content/AgentToolRoster.svelte b/frontend/src/lib/components/flows/content/AgentToolRoster.svelte new file mode 100644 index 0000000000..33abff2ba8 --- /dev/null +++ b/frontend/src/lib/components/flows/content/AgentToolRoster.svelte @@ -0,0 +1,172 @@ + + +{#snippet addToolButton()} + + {#snippet trigger()} + + {/snippet} + {#snippet content({ close })} + + (onAddTool?.(e.detail), close())} + on:insert={(e) => (onAddTool?.(e.detail), close())} + on:pickScript={(e) => ( + onAddTool?.({ + kind: e.detail.kind, + script: { + ...e.detail, + summary: e.detail.summary + ? e.detail.summary.replace(/\s/, '_').replace(/[^a-zA-Z0-9_]/g, '') + : e.detail.path.split('/').pop() + } + }), + close() + )} + on:pickMcpTool={() => (onAddTool?.({ kind: 'mcpTool' }), close())} + on:pickWebsearchTool={() => (onAddTool?.({ kind: 'websearchTool' }), close())} + on:pickAiAgentTool={() => (onAddTool?.({ kind: 'aiAgentTool' }), close())} + /> + {/snippet} + +{/snippet} + +{#if tools.length === 0} +
    +
    + {onAddTool ? 'No tools yet.' : emptyMessage} +
    + {#if onAddTool} + {@render addToolButton()} + {/if} +
    +{:else} +
    + + {#each tools as tool, i (i)} + {@const kind = toolKind(tool)} + {@const error = nameError(tool)} + +
    + + {#if onDeleteTool} +
    + {/each} +
    + {#if onAddTool} +
    {@render addToolButton()}
    + {/if} +{/if} diff --git a/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte b/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte index 194d19c6dc..6ccadc2ee5 100644 --- a/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte +++ b/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte @@ -15,6 +15,11 @@ forceTestTab?: Record highlightArg?: Record siblingToolNames?: string[] + /** See `FlowModuleComponent`: set when the tool belongs to a saved agent rather than to a + * step of this flow. */ + staticOnly?: boolean + /** See `FlowModuleComponent`: set where there is no graph to select a nested tool on. */ + noToolNavigation?: boolean } let { @@ -25,15 +30,28 @@ previousModule = undefined, forceTestTab, highlightArg, - siblingToolNames = undefined + siblingToolNames = undefined, + staticOnly = false, + noToolNavigation = false }: Props = $props() {#if isFlowModuleTool(tool)} + tool as FlowModule, + (v) => + (tool = { + ...tool, + ...v, + value: { tool_type: tool.value?.tool_type, ...v.value } + } as unknown as AgentTool) + } {parentModule} {previousModule} failureModule={false} @@ -45,6 +63,8 @@ forceTestTab={forceTestTab?.[tool.id]} highlightArg={highlightArg?.[tool.id]} isAgentTool={true} + {staticOnly} + {noToolNavigation} bind:toolDescription={tool.description} {siblingToolNames} /> diff --git a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte new file mode 100644 index 0000000000..33a15922f5 --- /dev/null +++ b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte @@ -0,0 +1,630 @@ + + + + +{#snippet unsetButton(spec: AgentFieldSpec)} + {#if !spec.core && !readOnly} + + {/snippet} + {#snippet menu({ close })} + +
    + {#each AGENT_FIELD_GROUPS as menuGroup (menuGroup.id)} + {@const groupCandidates = candidates.filter((spec) => spec.group === menuGroup.id)} + {#if groupCandidates.length > 0} +
    + {menuGroup.label} +
    + {#each groupCandidates as spec (spec.key)} + + {/each} + {/if} + {/each} +
    + {/snippet} + + {/if} +{/snippet} + +
    + + diff --git a/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte b/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte index 4aede5e3d1..979717dae1 100644 --- a/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte +++ b/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte @@ -1,7 +1,6 @@ - { - showChatModeWarning = false - chatInputEnabled = false - }} -> -

    - Enabling Chat Mode will replace all existing flow inputs with a single - user_message - parameter. -

    -

    - Your current input configuration will be lost. Are you sure you want to continue? -

    -
    + +{#snippet inputsEditButton(open: boolean, toggle: () => void)} + + + {#snippet children({ item })} + + + {/snippet} + {/if}
    {/if} @@ -705,11 +806,15 @@
    {#if flowStore.val.value?.chat_input_enabled}
    - {#if showAdditionalInputs} -
    + {#if chatPanelTab === 'inputs'} + +
    {#snippet openEditTab()} - + {@render inputsAddTrigger()} {/snippet} {/snippet}
    {/if} - + +
    + +
    {:else}
    @@ -825,22 +929,9 @@
    {#snippet close_button()} -
    @@ -890,16 +981,11 @@ onAddNew={(argName) => { handleEditSchema('inputEditor') editableSchemaForm?.openField(argName) - refreshFlowStateStore(flowStore) + refreshStateStore(flowStore) }} > {#snippet trigger()} -
    - -
    + {@render inputsAddTrigger()} {/snippet} {/if} diff --git a/frontend/src/lib/components/flows/content/FlowInputsFlow.svelte b/frontend/src/lib/components/flows/content/FlowInputsFlow.svelte index 4938b470af..2d4b124694 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsFlow.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsFlow.svelte @@ -3,13 +3,15 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import SearchItems from '$lib/components/SearchItems.svelte' import { FlowService, type Flow } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { emptyString } from '$lib/utils' import { createEventDispatcher, getContext, untrack } from 'svelte' import type { FlowEditorContext } from '../types' import { flip } from 'svelte/animate' import { fade } from 'svelte/transition' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { children?: import('svelte').Snippet } @@ -17,7 +19,7 @@ let { children }: Props = $props() const flowEditorContext = getContext('FlowEditorContext') - let opWs = $derived(flowEditorContext?.opWorkspace?.() ?? $workspaceStore) + let opWs = $derived(flowEditorContext?.opWorkspace?.() ?? $operatingWorkspace) // export let failureModule: boolean const dispatch = createEventDispatcher() diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index cf9d008a27..ae28a377c4 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -7,20 +7,13 @@ import { sendUserToast } from '$lib/toast' import FlowScriptPickerQuick from '../pickers/FlowScriptPickerQuick.svelte' import { defaultScriptLanguages, processInlineLangs } from '$lib/scripts' - import { - defaultScripts, - enterpriseLicense, - hubBaseUrlStore, - userStore, - workspaceStore - } from '$lib/stores' + import { defaultScripts, enterpriseLicense, hubBaseUrlStore } from '$lib/stores' import type { SupportedLanguage } from '$lib/common' - import { createEventDispatcher, getContext, onDestroy, onMount, untrack } from 'svelte' + import { createEventDispatcher, getContext, untrack } from 'svelte' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' import { type Script, type ScriptLang, type HubScriptKind } from '$lib/gen' import ListFiltersQuick from '$lib/components/home/ListFiltersQuick.svelte' import { ExternalLink, Folder, User, X } from 'lucide-svelte' - import type { FlowEditorContext } from '../../flows/types' import { fade } from 'svelte/transition' import { flip } from 'svelte/animate' import { Button } from '$lib/components/common' @@ -35,6 +28,14 @@ canHaveApproval, canHaveFailure } from '$lib/script_helpers' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) const dispatch = createEventDispatcher() @@ -66,8 +67,8 @@ refreshCount = 0 }: Props = $props() - if ($workspaceStore && cachedOwners?.[$workspaceStore]) { - owners = cachedOwners[$workspaceStore] + if ($operatingWorkspace && cachedOwners?.[$operatingWorkspace]) { + owners = cachedOwners[$operatingWorkspace] } type HubCompletion = { path: string @@ -87,8 +88,6 @@ let hubCompletions: HubCompletion[] = $state([]) - const { insertButtonOpen } = getContext('FlowEditorContext') - let selected: { kind: 'owner' | 'integrations'; name: string | undefined } | undefined = $state(undefined) @@ -221,13 +220,6 @@ selectedByKeyboard = index } - onMount(() => { - $insertButtonOpen = true - }) - - onDestroy(() => { - $insertButtonOpen = false - }) let langs = $derived( processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) @@ -255,6 +247,7 @@ // on indices that render nothing. let showAiRows = $derived( !disableAi && + !$copilotInfo.workspaceDisabled && funcDesc?.length > 0 && kind != 'failure' && kind != 'preprocessor' && @@ -409,7 +402,7 @@
    New {selectedKind != 'script' ? selectedKind + ' ' : ''}script
    - {#if $userStore?.is_admin || $userStore?.is_super_admin} + {#if actingUser?.is_admin || actingUser?.is_super_admin} {#if !openScriptSettings} @@ -101,83 +266,83 @@ {/if} -
    - -
    - {#if deploymentInProgress} - - {/if} - {#if manager.isLoadingMessages} -
    - -
    - {:else if manager.messages.length === 0} -
    - -

    Start a conversation

    -

    Send a message to run the flow and see the results

    -
    +{#snippet emptyHint()} +
    + {#if chatHost.state.loadingMessages} + {:else} -
    - {#each manager.messages as message (message.id)} - - {/each} - {#if manager.isWaitingForResponse} -
    - - Processing... -
    - {/if} -
    - {/if} -
    - - -
    - {#if additionalInputsSchema} -
    -
    - - {#if hasMissingRequired} - - {/if} + +

    Start a conversation

    +

    Send a message to run the flow and see the results

    + {#if !emptyString(description)} +
    +
    -
    + {/if} {/if} -
    - { - if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { - e.preventDefault() - handleSendMessage() - } - }} - showCancelButton={manager.isWaitingForResponse || manager.isLoading} - onCancel={() => manager.cancelCurrentJob()} - sendTitle={deploymentInProgress ? 'Deployment in progress' : 'Send message (Enter)'} - /> -
    +{/snippet} + +{#snippet footerSettings()} + {#if modalSchema} +
    + + {#if modalMissingRequired} + + {/if} +
    + {/if} + {#if modelWiring && showModelButton} + + + {/if} +{/snippet} + + +
    0} + class:min-h-64={chatHost.displayMessages.length === 0} +> + {}} + deletePastChat={() => {}} + saveAndClear={() => {}} + />
    diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts deleted file mode 100644 index 7cf2c8d6f7..0000000000 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ /dev/null @@ -1,684 +0,0 @@ -import type { FlowConversation, FlowConversationMessage } from '$lib/gen/types.gen' -import { FlowConversationsService, JobService } from '$lib/gen' -import { sendUserToast } from '$lib/toast' -import { waitJob } from '$lib/components/waitJob' -import { tick } from 'svelte' -import InfiniteList from '$lib/components/InfiniteList.svelte' -import { workspaceStore, userStore } from '$lib/stores' -import { get } from 'svelte/store' -import { parseStreamDeltas } from '$lib/components/chat/utils' -import { randomUUID } from '$lib/utils/uuid' - -export interface ChatMessage extends FlowConversationMessage { - loading?: boolean - streaming?: boolean -} - -export interface ConversationWithDraft extends FlowConversation { - isDraft?: boolean -} - -export class FlowChatManager { - // State - messages = $state([]) - inputMessage = $state('') - isLoading = $state(false) - isLoadingMessages = $state(false) - isWaitingForResponse = $state(false) - messagesContainer = $state(undefined) - inputElement = $state(undefined) - page = $state(1) - hasMoreMessages = $state(false) - loadingMoreMessages = $state(false) - currentEventSource = $state(undefined) - pollingInterval = $state | undefined>(undefined) - currentJobId = $state(undefined) - conversations = $state([]) - deletingConversationId = $state(undefined) - isSidebarExpanded = $state(false) - selectedConversationId = $state(undefined) - conversationListComponent = $state(undefined) - - // Private state - #conversationsCache = $state>({}) - #scrollTimeout: ReturnType | undefined = undefined - #perPage = 50 - - // Options - #onRunFlow?: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise - #useStreaming = $state(false) - #path = $state(undefined) - - // When the flow editor runs as an AI-session live editor, it acts on a workspace - // that can differ from the nav store. FlowChat.svelte wires this to - // FlowEditorContext.opWorkspace so workspace-scoped calls hit the acting workspace. - operatingWorkspace?: () => string | undefined - - #workspace(): string | undefined { - return this.operatingWorkspace?.() ?? get(workspaceStore) - } - - initialize( - onRunFlow: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise, - path: string, - useStreaming: boolean = false - ) { - this.#onRunFlow = onRunFlow - this.#path = path - this.#useStreaming = useStreaming - } - - updateConversationId(conversationId: string | undefined) { - this.selectedConversationId = conversationId - } - - cleanup() { - if (this.currentEventSource) { - this.currentEventSource.close() - this.currentEventSource = undefined - } - this.stopPolling() - this.isLoading = false - this.isWaitingForResponse = false - this.currentJobId = undefined - } - - // Public methods for component to call - fillInputMessage(message: string) { - this.inputMessage = message - } - - focusInput() { - this.inputElement?.focus() - } - - clearMessages() { - this.messages = [] - this.inputMessage = '' - this.page = 1 - } - - async createConversation({ clearMessages = true }: { clearMessages?: boolean }) { - // Check if there's already a draft conversation - const existingDraft = this.conversations.find((c) => c.isDraft) - if (existingDraft) { - // Select the existing draft instead of creating a new one - this.selectedConversationId = existingDraft.id - this.clearMessages() - return existingDraft.id - } - const newConversationId = randomUUID() - this.selectedConversationId = newConversationId - - // Create a new conversation object and add it to the top of the list - const newConversation: ConversationWithDraft = { - id: newConversationId, - workspace_id: this.#workspace()!, - flow_path: this.#path!, - title: 'New chat', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - created_by: get(userStore)!.username!, - isDraft: true - } - - // Prepend to conversations list - this.conversations = [newConversation, ...this.conversations] - // Clear messages in the chat interface - if (clearMessages) { - this.clearMessages() - } - this.focusInput() - - return newConversationId - } - - setupInfiniteList() { - this.conversationListComponent?.setLoader((page, perPage) => - this.loadConversations(page, perPage) - ) - this.conversationListComponent?.setDeleteItemFn((id) => this.deleteConversation(id)) - } - - async selectConversation(conversationId: string, isDraft?: boolean) { - this.selectedConversationId = conversationId - // Load conversation messages into chat interface - if (isDraft) { - // For draft conversations, just clear messages (don't try to load from backend) - this.clearMessages() - } else { - // For persisted conversations, load messages from backend - await this.loadConversationMessages(conversationId) - } - } - - async refreshConversations() { - await this.conversationListComponent?.loadData('forceRefresh') - } - - // Only used by InfiniteList - private async deleteConversation(conversationId: string) { - try { - this.deletingConversationId = conversationId - await FlowConversationsService.deleteFlowConversation({ - workspace: this.#workspace()!, - conversationId - }) - if (this.selectedConversationId === conversationId) { - this.selectedConversationId = undefined - this.clearMessages() - } - sendUserToast('Conversation deleted successfully') - } catch (error) { - console.error('Failed to delete conversation:', error) - sendUserToast('Failed to delete conversation', true) - throw error - } finally { - this.deletingConversationId = undefined - } - } - - async cancelCurrentJob() { - if (!this.#workspace()) { - return - } - - try { - if (this.currentJobId) { - await JobService.cancelQueuedJob({ - workspace: this.#workspace()!, - id: this.currentJobId, - requestBody: {} - }) - sendUserToast(`Job ${this.currentJobId} cancelled`) - } - } catch (error) { - console.error('Error cancelling job:', error) - sendUserToast('Could not cancel job', true) - } finally { - this.cleanup() - } - } - - async loadConversationMessages(conversationId?: string) { - this.page = 1 - await this.loadMessages(true, conversationId) - } - - // Only used by InfiniteList - private async loadConversations(page: number, perPage: number) { - if (!this.#workspace() || !this.#path) return [] - - try { - const response = await FlowConversationsService.listFlowConversations({ - workspace: this.#workspace()!, - flowPath: this.#path, - page: page, - perPage: perPage - }) - - return response - } catch (error) { - console.error('Failed to load conversations:', error) - sendUserToast('Failed to load conversations', true) - return [] - } - } - - // Message loading - private async loadMessages(reset: boolean, conversationId?: string) { - let conversationIdToUse = conversationId ?? this.selectedConversationId - if (!this.#workspace() || !conversationIdToUse) return - - if (reset) { - if (this.#conversationsCache[conversationIdToUse]) { - this.messages = this.#conversationsCache[conversationIdToUse] - return - } - this.isLoadingMessages = true - } else { - this.loadingMoreMessages = true - } - - const pageToFetch = reset ? 1 : this.page + 1 - - try { - const previousScrollHeight = this.messagesContainer?.scrollHeight || 0 - - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationIdToUse, - page: pageToFetch, - perPage: this.#perPage - }) - - if (reset) { - this.#conversationsCache[conversationIdToUse] = response - this.messages = response - this.isLoadingMessages = false - await new Promise((resolve) => setTimeout(resolve, 100)) - this.scrollToBottom() - } else { - this.messages = [...response, ...this.messages] - this.page = pageToFetch - // Restore scroll position - await new Promise((resolve) => setTimeout(resolve, 50)) - if (this.messagesContainer) { - this.messagesContainer.scrollTop = - this.messagesContainer.scrollHeight - previousScrollHeight - } - } - - this.hasMoreMessages = response.length === this.#perPage - } catch (error) { - console.error('Failed to load messages:', error) - sendUserToast('Failed to load messages: ' + error) - } finally { - this.isLoadingMessages = false - this.loadingMoreMessages = false - } - } - - handleScroll = () => { - if (this.#scrollTimeout) clearTimeout(this.#scrollTimeout) - - this.#scrollTimeout = setTimeout(() => { - if (!this.messagesContainer || !this.hasMoreMessages || this.loadingMoreMessages) return - - if (this.messagesContainer.scrollTop <= 10) { - this.loadMessages(false) - } - }, 200) - } - - scrollToBottom() { - if (this.messagesContainer) { - this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight - } - } - - private scrollToUserMessage(messageId: string) { - if (!this.messagesContainer) return - const messageElement = this.messagesContainer.querySelector(`[data-message-id="${messageId}"]`) - if (messageElement) { - messageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }) - } - } - - private getLastPersistedMessageSeq() { - for (let i = this.messages.length - 1; i >= 0; i--) { - const message = this.messages[i] - if (!message.id.startsWith('temp-')) { - return message.created_seq - } - } - - return undefined - } - - // Polling - private async pollJobResult(jobId: string) { - try { - await waitJob(jobId, this.#workspace()) - } catch (error) { - console.error('Error polling job result:', error) - } finally { - // Do a final poll to get all messages from database - try { - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - } catch {} - this.cleanup() - } - } - - private async pollConversationMessages( - conversationId: string, - options?: { isNewConversation?: boolean; removeTempMessages?: boolean } - ) { - if (!this.#workspace()) return - - try { - const lastSeq = this.getLastPersistedMessageSeq() - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationId, - page: 1, - perPage: 50, - afterSeq: lastSeq - }) - - if (options?.isNewConversation) { - await this.refreshConversations() - } - - const filteredResponse = response.filter((msg) => msg.message_type !== 'user') - for (const msg of filteredResponse) { - if (!this.messages.find((m) => m.id === msg.id)) { - this.messages = [...this.messages, msg] - } - } - - // Only remove temporary messages when explicitly requested (e.g., after job completion) - // During streaming, we keep temp messages to avoid them disappearing due to race conditions - if (options?.removeTempMessages) { - this.messages = this.messages.filter( - (msg) => !msg.id.startsWith('temp-') || msg.message_type === 'user' - ) - } - } catch (error) { - console.error('Polling error:', error) - } - } - - private startPolling(conversationId: string, isNewConversation?: boolean) { - if (this.pollingInterval) return - this.pollingInterval = setInterval(() => { - this.pollConversationMessages(conversationId, { isNewConversation }) - }, 500) // Poll every 0.5 seconds - setTimeout( - () => { - this.stopPolling() - }, - 2 * 60 * 1000 - ) // Stop polling after 2 minutes - } - - private stopPolling() { - if (this.pollingInterval) { - clearInterval(this.pollingInterval) - this.pollingInterval = undefined - } - } - - // Message sending - async sendMessage(additionalInputs?: Record) { - if (!this.inputMessage.trim() || this.isLoading) return - - const isNewConversation = this.messages.length === 0 - - // Reset state for new message - this.stopPolling() - - // Generate a new conversation ID if we don't have one - let currentConversationId = this.selectedConversationId - if (!this.selectedConversationId) { - const newConversationId = await this.createConversation({ clearMessages: false }) - currentConversationId = newConversationId - } - - if (!currentConversationId) { - console.error('No conversation ID found') - return - } - - // Invalidate the conversation cache - delete this.#conversationsCache[currentConversationId] - - const userMessage: ChatMessage = { - id: `temp-${randomUUID()}`, - content: this.inputMessage.trim(), - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'user', - conversation_id: currentConversationId - } - - this.messages = [...this.messages, userMessage] - const messageContent = this.inputMessage.trim() - this.inputMessage = '' - this.isLoading = true - this.isWaitingForResponse = true - - try { - await tick() - this.scrollToUserMessage(userMessage.id) - - if (this.#useStreaming && this.#path) { - await this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } else { - await this.handlePollingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } - } catch (error) { - console.error('Error running flow:', error) - sendUserToast('Failed to run flow: ' + error, true) - } finally { - if (!this.#useStreaming) { - this.isLoading = false - } - } - - await tick() - this.focusInput() - } - - private async handleStreamingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - // Close any existing EventSource - if (this.currentEventSource) { - this.currentEventSource.close() - } - - // Track stream state for this message - let accumulatedContent = '' - let assistantMessageId = '' - let isCompleted = false - - try { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - - // Build the EventSource URL - const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` - const url = new URL(streamUrl, window.location.origin) - url.searchParams.set('poll_delay_ms', '50') - url.searchParams.set('fast', 'true') - url.searchParams.set('only_result', 'true') - // Create EventSource connection - const eventSource = new EventSource(url.toString()) - this.currentEventSource = eventSource - - // start polling - this.startPolling(currentConversationId, isNewConversation) - - eventSource.onmessage = async (event) => { - try { - const data = JSON.parse(event.data) - const type = data.type - - // Handle timeout - reconnect to SSE - if (type === 'timeout') { - eventSource.close() - this.currentEventSource = undefined - // Reconnect - this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - return - } - - // Handle ping - just ignore - if (type === 'ping') { - return - } - - // Handle error - if (type === 'error') { - eventSource.close() - this.currentEventSource = undefined - console.error('SSE error:', data) - sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) - this.cleanup() - return - } - - // Handle not found - if (type === 'not_found') { - eventSource.close() - this.currentEventSource = undefined - console.error('Job not found') - sendUserToast('Job not found', true) - this.cleanup() - return - } - - if (type === 'update') { - if (data.flow_stream_job_id) { - this.currentJobId = data.flow_stream_job_id - } - // Process new stream content - if (data.new_result_stream) { - // Stop polling since we are receiving last step streaming - this.stopPolling() - const { - type, - content: newContent, - success - } = parseStreamDeltas(data.new_result_stream) - accumulatedContent += newContent - - // Create tool message if type is tool_result - if (type === 'tool_result') { - // set last message streaming to false - this.messages = this.messages.map((msg) => - msg.id === this.messages[this.messages.length - 1].id - ? { ...msg, streaming: false } - : msg - ) - - this.messages = [ - ...this.messages, - { - id: 'temp-' + randomUUID(), - content: newContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'tool', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: false, - success - } - ] - // Reset assistant message ID since we are creating a tool message - assistantMessageId = '' - accumulatedContent = '' - } - - // Create message on first content - else if ( - type === 'message' && - assistantMessageId.length === 0 && - accumulatedContent.length > 0 - ) { - assistantMessageId = 'temp-' + randomUUID() - this.messages = [ - ...this.messages, - { - id: assistantMessageId, - content: accumulatedContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'assistant', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: true - } - ] - } else { - // Update existing message - this.messages = this.messages.map((msg) => - msg.id === assistantMessageId ? { ...msg, content: accumulatedContent } : msg - ) - } - } - - // Handle completion - if (data.completed) { - isCompleted = true - // Do a final poll to get all messages from database - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - this.cleanup() - } - } - } catch (error) { - console.error('Error processing stream event:', error) - } - } - - eventSource.onerror = (error) => { - if (isCompleted) return - console.error('EventSource error:', error) - sendUserToast('Stream error occurred', true) - this.cleanup() - } - } catch (error) { - console.error('Stream connection error:', error) - sendUserToast('Failed to connect to stream', true) - this.cleanup() - } - } - - private async handlePollingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - - // Store the current job ID so it can be cancelled - this.currentJobId = jobId - - if (isNewConversation) { - await this.refreshConversations() - } - - // Start polling for intermediate messages in non-streaming mode too - this.startPolling(currentConversationId) - this.pollJobResult(jobId) - } -} - -export const createFlowChatManager = () => new FlowChatManager() diff --git a/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte b/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte new file mode 100644 index 0000000000..f27e14749e --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte @@ -0,0 +1,299 @@ + + +{#snippet providerSummary()} + {#if resourcePath} + {resourcePath} + {/if} +{/snippet} + +{#if resourceEditable} + { + resourcesVersion++ + if (e.detail) { + pendingResourcePath = e.detail + } + }} + /> +{/if} + + diff --git a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte index 2f09bcebe1..d2ea5c5061 100644 --- a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte @@ -1,142 +1,389 @@
    -
    +
    - + +
    + + {#if canFilterKind} + + + {#snippet trigger()} + +
    - {#if !manager.isSidebarExpanded} + {#if !expanded}
    {/if} -
    +
    + {#if draftShown && expanded} +
    + +
    + {/if} - {#snippet customRow({ item: conversation, hover })} - {#if manager.isSidebarExpanded} + {#snippet customRow({ item: conversation })} + {#if expanded}
    - + onClick={() => { + draft = false + chat.selectConversation(conversation.id) + }} + selected={chatState.conversationId === conversation.id} + btnClasses="transition-all duration-150 group" + > + + {getConversationTitle(conversation)} + + + +
    e.stopPropagation()} + > + rowActions(conversation)} size="xs" /> +
    + + {/if}
    {/if} {/snippet} {#snippet empty()} -
    -

    No conversations yet

    -
    + {#if !draftShown} +
    +

    No conversations yet

    +
    + {/if} {/snippet}
    diff --git a/frontend/src/lib/components/flows/conversations/agentAttachmentInput.test.ts b/frontend/src/lib/components/flows/conversations/agentAttachmentInput.test.ts new file mode 100644 index 0000000000..99bee0044c --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentAttachmentInput.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it } from 'vitest' +import { + agentSteps, + attachmentsTargetFor, + flowInputRef, + resolveAgentChatInputs +} from './agentAttachmentInput' +import type { FlowModule } from '$lib/gen' + +/** An agent step with the given input transforms, as the editor stores them. */ +function agentWith(input_transforms: Record, id = 'a'): FlowModule { + return { + id, + value: { type: 'aiagent', tools: [], input_transforms } + } as unknown as FlowModule +} + +describe('agentSteps', () => { + it('finds agents inside loops and branches', () => { + const modules = [ + { id: 'loop', value: { type: 'forloopflow', modules: [agentWith({}, 'in-loop')] } }, + { + id: 'branch', + value: { + type: 'branchone', + default: [agentWith({}, 'in-default')], + branches: [{ modules: [agentWith({}, 'in-branch')] }] + } + }, + { + id: 'all', + value: { type: 'branchall', branches: [{ modules: [agentWith({}, 'in-all')] }] } + } + ] as unknown as FlowModule[] + expect(agentSteps(modules).map((m) => m.id)).toEqual([ + 'in-loop', + 'in-default', + 'in-branch', + 'in-all' + ]) + }) + + // The graph walks an agent's tools as child steps; a tool agent's inputs belong to the + // agent that calls it, not to the chat. + it("ignores an agent carried as another agent's tool", () => { + const parent = agentWith({}) + ;(parent.value as any).tools = [ + { id: 'summarize', value: { tool_type: 'flowmodule', type: 'aiagent', tools: [] } } + ] + expect(agentSteps([parent]).map((m) => m.id)).toEqual(['a']) + }) +}) + +describe('flowInputRef', () => { + it('names the one input an expression reads, however it reshapes it', () => { + expect( + flowInputRef({ + type: 'javascript', + expr: '(flow_input.files || []).map(f => ({ bucket: f.storage, key: f.s3 }))' + }) + ).toBe('files') + expect(flowInputRef({ type: 'javascript', expr: 'flow_input?.docs' })).toBe('docs') + }) + + // A loop step reads its iteration from `flow_input.iter`, which is not a flow input. + it('ignores names the flow does not declare, such as a loop iteration', () => { + const transform = { + type: 'javascript' as const, + expr: 'flow_input.files.filter((_, i) => i === flow_input.iter.index)' + } + expect(flowInputRef(transform)).toBeUndefined() + expect(flowInputRef(transform, { files: {} })).toBe('files') + }) + + it('counts real reads only, in any access form and in a statement body', () => { + const ref = (expr: string) => + flowInputRef({ type: 'javascript', expr }, { files: {}, docs: {} }) + expect(ref("flow_input['files']")).toBe('files') + expect(ref('/* flow_input.docs */ flow_input?.files')).toBe('files') + expect(ref("'flow_input.docs' + flow_input.files")).toBe('files') + expect(ref('results.a.flow_input.docs ?? flow_input.files')).toBe('files') + expect(ref('const f = flow_input.files\nreturn f')).toBe('files') + expect(ref('flow_input.files.concat(')).toBeUndefined() + }) + + it('names nothing for two inputs or a static value', () => { + expect( + flowInputRef({ type: 'javascript', expr: '[...flow_input.a, ...flow_input.b]' }) + ).toBeUndefined() + expect(flowInputRef({ type: 'static', value: [] })).toBeUndefined() + }) +}) + +describe('resolveAgentChatInputs', () => { + const schema = { properties: { files: { type: 'array' } }, required: [] } + const reader = (name: string) => + agentWith({ user_attachments: { type: 'javascript', expr: `flow_input.${name}` } }) + // Every agent step carries a placeholder transform for each key of AI_AGENT_SCHEMA + // (loadSchemaFromModule writes them back onto the module), so an agent that reads + // nothing must not be mistaken for one reading a different input. + const seeded = () => agentWith({ user_attachments: { type: 'static', value: undefined } }) + + it('promotes the input one agent reads', () => { + expect(resolveAgentChatInputs([reader('files')], schema).map((i) => i.name)).toEqual(['files']) + }) + + it('still promotes it when another agent leaves the field unwired', () => { + expect(resolveAgentChatInputs([reader('files'), seeded()], schema).map((i) => i.name)).toEqual([ + 'files' + ]) + }) + + it('promotes nothing when two agents read different inputs', () => { + const twoInputs = { + properties: { files: { type: 'array' }, docs: { type: 'array' } }, + required: [] + } + expect(resolveAgentChatInputs([reader('files'), reader('docs')], twoInputs)).toEqual([]) + }) + + it('promotes the input an agent inside a loop reads next to its iteration', () => { + const loop = { + id: 'loop', + value: { + type: 'forloopflow', + modules: [ + agentWith({ + user_attachments: { + type: 'javascript', + expr: 'flow_input.files.filter((_, i) => i === flow_input.iter.index)' + } + }) + ] + } + } as unknown as FlowModule + expect(resolveAgentChatInputs([loop], schema).map((i) => i.name)).toEqual(['files']) + }) + + it('gives no say to an agent that only mentions flow_input in a comment', () => { + const commented = agentWith({ + user_attachments: { type: 'javascript', expr: '// flow_input.docs\nresults.a.files' } + }) + expect(resolveAgentChatInputs([reader('files'), commented], schema).map((i) => i.name)).toEqual( + ['files'] + ) + }) + + it('promotes nothing for an input the schema does not declare', () => { + expect(resolveAgentChatInputs([reader('missing')], schema)).toEqual([]) + }) +}) + +describe('attachmentsTargetFor', () => { + const input = (property: Record) => + ({ name: 'files', key: 'user_attachments', property }) as any + + it('takes a list of s3 files, and says it holds several', () => { + expect( + attachmentsTargetFor(input({ type: 'array', items: { resourceType: 's3object' } })) + ).toEqual({ name: 'files', multiple: true }) + }) + + it('takes a single s3 file', () => { + expect(attachmentsTargetFor(input({ format: 'resource-s3_object' }))).toEqual({ + name: 'files', + multiple: false + }) + }) + + // The transform can build the s3 object itself, promoting an input that holds a key + // rather than a file. Uploading into it would write an object where a string is declared. + it('offers no paperclip where the input cannot hold a file', () => { + expect(attachmentsTargetFor(input({ type: 'string' }))).toBeUndefined() + expect( + attachmentsTargetFor(input({ type: 'array', items: { type: 'string' } })) + ).toBeUndefined() + expect(attachmentsTargetFor(undefined)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/agentAttachmentInput.ts b/frontend/src/lib/components/flows/conversations/agentAttachmentInput.ts new file mode 100644 index 0000000000..fcae1d6619 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentAttachmentInput.ts @@ -0,0 +1,185 @@ +import type { FlowModule, InputTransform } from '$lib/gen' +import { parse, parseExpressionAt } from 'acorn' + +/** + * The flow's own AI agent steps, including those inside loops and branches. An agent carried + * as another agent's tool is left out, unlike in the graph (flowTree.ts): its inputs come from + * the agent calling it, not from the chat. + */ +export function agentSteps(modules: FlowModule[] | undefined): FlowModule[] { + const found: FlowModule[] = [] + const walk = (mods: FlowModule[]) => { + for (const module of mods) { + const value = module.value as any + if (value?.type === 'aiagent') { + found.push(module) + continue + } + if (value?.type === 'forloopflow' || value?.type === 'whileloopflow') { + walk(value.modules ?? []) + } else if (value?.type === 'branchone') { + walk(value.default ?? []) + for (const branch of value.branches ?? []) walk(branch.modules ?? []) + } else if (value?.type === 'branchall') { + for (const branch of value.branches ?? []) walk(branch.modules ?? []) + } + } + } + walk(modules ?? []) + return found +} + +/** + * AI agent inputs the chat composer drives, through the flow input the author wired to each: + * the composer writes run inputs, never the flow. Only what the person chatting owns turn to + * turn belongs here. `system_prompt`, `temperature` and the like shape the agent for everyone + * who runs the flow, so they stay in the Configure-inputs modal. + */ +export const AGENT_CHAT_INPUT_KEYS = ['user_attachments'] as const + +export type AgentChatInputKey = (typeof AGENT_CHAT_INPUT_KEYS)[number] + +/** The key that rides one message rather than the conversation, cleared on send. */ +export const PER_TURN_AGENT_CHAT_INPUT_KEY: AgentChatInputKey = 'user_attachments' + +export type AgentChatInput = { + /** Flow input property feeding the agent field. */ + name: string + key: AgentChatInputKey + /** The flow input's own schema entry. */ + property: Record +} + +/** + * The `flow_input` properties an expression reads, from its syntax tree rather than its text, + * so a mention in a comment, a string or a nested path (`results.a.flow_input.x`) is not a read. + * A transform may be a statement body with `return`. Undefined when it does not parse. + */ +export function flowInputReads(expr: string): Set | undefined { + let root: unknown + try { + root = parseExpressionAt(`(\n${expr}\n)`, 0, { ecmaVersion: 'latest' }) + } catch { + try { + root = parse(expr, { + ecmaVersion: 'latest', + allowReturnOutsideFunction: true, + allowAwaitOutsideFunction: true + }) + } catch { + return undefined + } + } + const names = new Set() + const visit = (node: any) => { + if (!node || typeof node !== 'object') return + if (Array.isArray(node)) return node.forEach(visit) + if ( + node.type === 'MemberExpression' && + node.object?.type === 'Identifier' && + node.object.name === 'flow_input' + ) { + if (!node.computed && node.property?.type === 'Identifier') names.add(node.property.name) + else if (node.computed && typeof node.property?.value === 'string') + names.add(node.property.value) + } + for (const key in node) if (key !== 'type') visit(node[key]) + } + visit(root) + return names +} + +/** + * The flow input a transform reads, when it reads exactly one. The expression may reshape it + * (`(flow_input.files || []).map(...)`) and still counts. With `declared`, only the flow's own + * inputs count, so a loop's `flow_input.iter` does not make a read ambiguous. + */ +export function flowInputRef( + transform: InputTransform | undefined, + declared?: Record +): string | undefined { + if (transform?.type !== 'javascript') return undefined + const reads = flowInputReads(transform.expr) + if (!reads) return undefined + const names = declared ? [...reads].filter((name) => name in declared) : [...reads] + return names.length === 1 ? names[0] : undefined +} + +/** Whether a schema entry holds an s3 file, as the flow input editor recognises one. */ +function holdsS3File(property: Record | undefined): boolean { + return ( + property?.format === 'resource-s3_object' || + property?.resourceType === 's3object' || + property?.resourceType === 's3_object' + ) +} + +/** The flow input the composer's attachments feed, whether it holds a list, and whether the + * flow requires it (so a message without a file cannot run). */ +export type AttachmentsTarget = { name: string; multiple: boolean; required?: boolean } + +/** + * Where the composer's attachments go: the promoted input, only when its schema holds s3 + * objects. A transform may build the s3 object from a plain string input, and writing + * `{ s3, filename }` into that input would fail at run time. + */ +export function attachmentsTargetFor( + input: AgentChatInput | undefined +): AttachmentsTarget | undefined { + if (!input) return undefined + if (holdsS3File(input.property)) return { name: input.name, multiple: false } + return input.property?.type === 'array' && holdsS3File(input.property.items) + ? { name: input.name, multiple: true } + : undefined +} + +/** + * The flow inputs that an AI agent step reads directly into one of its chat-relevant + * fields. Several agents may read the same flow input; it is promoted once. + */ +export function resolveAgentChatInputs( + modules: FlowModule[] | undefined, + additionalInputsSchema: Record | undefined +): AgentChatInput[] { + const properties = additionalInputsSchema?.properties + if (!modules || !properties) return [] + + // One input per key, and only when every agent reading that key reads the same one: + // the composer writes a single flow input, so promoting one of two would feed one + // agent and leave the other with nothing — while hiding both from the modal, where + // the reader could at least have filled them in. + const namesPerKey = new Map>() + for (const module of agentSteps(modules)) { + const transforms = (module.value as any).input_transforms ?? {} + for (const key of AGENT_CHAT_INPUT_KEYS) { + const transform = transforms[key] + // An agent that feeds the key from anything but a flow input — a literal, another + // step's result, or the empty placeholder every agent step carries for the keys of + // AI_AGENT_SCHEMA — is not reading an input, so it has no say in which one the + // composer drives. An expression that does not parse is kept, as unreadable. + if (transform?.type !== 'javascript') continue + const reads = flowInputReads(transform.expr) + if (reads ? reads.size === 0 : !transform.expr.includes('flow_input')) continue + // Reading no declared input, or two of them, names none: this agent reads something + // the composer cannot drive, which is what disagreement means here. + const usable = flowInputRef(transform, properties) + const names = namesPerKey.get(key) ?? new Set() + names.add(usable) + namesPerKey.set(key, names) + } + } + + const keyOf = new Map() + for (const [key, names] of namesPerKey) { + if (names.size !== 1) continue + const name = [...names][0] + if (name === undefined || keyOf.has(name)) continue + keyOf.set(name, key) + } + + return [...keyOf.entries()].map(([name, key]) => ({ + name, + key, + property: properties[name] + })) +} diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts new file mode 100644 index 0000000000..00561330d0 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts @@ -0,0 +1,501 @@ +import { describe, expect, it } from 'vitest' +import { + agentModelGap, + agentModelWiringInputs, + composerOwnedInputs, + parseProviderTransform, + resolveAgentModelWiring, + showsModelButton, + withoutRejectedEffort +} from './agentChatInputs' +import type { FlowModule } from '$lib/gen' + +function agent(expr: string): FlowModule { + return { + id: 'a', + value: { + type: 'aiagent', + tools: [], + input_transforms: { provider: { type: 'javascript', expr } } + } + } as unknown as FlowModule +} + +describe('parseProviderTransform', () => { + it('reads a whole-object reference', () => { + expect(parseProviderTransform({ type: 'javascript', expr: 'flow_input.model' })).toEqual({ + whole: 'model', + fields: {}, + fixed: {} + }) + }) + + it('splits a partial expression into wired and fixed fields', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: "{ kind: 'anthropic', resource: '$res:u/admin/claude', model: flow_input.model, reasoning_effort: flow_input.thinking }" + }) + expect(wiring).toEqual({ + fields: { model: 'model', reasoning_effort: 'thinking' }, + fixed: { kind: 'anthropic', resource: '$res:u/admin/claude' } + }) + }) + + // The regression this detector exists for: counting flow_input references would read + // this as one input carrying the whole provider object, and the composer would write + // {kind, resource, model} into an input the expression uses as the model name. + it('does not mistake a single-reference partial expression for a whole-object one', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: "{ kind: 'anthropic', resource: '$res:u/admin/claude', model: flow_input.model }" + }) + expect(wiring?.whole).toBeUndefined() + expect(wiring?.fields).toEqual({ model: 'model' }) + }) + + // The flow editor's JS field commonly holds a parenthesised object, which is how an + // author writes one without it reading as a block. + it('accepts a parenthesised object expression', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: `({ + "kind": "anthropic", + "resource": "$res:u/admin/anthropic_windmill_codegen", + "model": "claude-sonnet-5", + "reasoning_effort": flow_input.thinking +})` + }) + expect(wiring).toEqual({ + fields: { reasoning_effort: 'thinking' }, + fixed: { + kind: 'anthropic', + resource: '$res:u/admin/anthropic_windmill_codegen', + model: 'claude-sonnet-5' + } + }) + }) + + it('treats a static provider as entirely fixed', () => { + expect( + parseProviderTransform({ + type: 'static', + value: { kind: 'openai', resource: '$res:u/admin/oai', model: 'gpt-5.6' } + }) + ).toEqual({ + fields: {}, + fixed: { kind: 'openai', resource: '$res:u/admin/oai', model: 'gpt-5.6' } + }) + }) + + it.each([ + ['{ ...base, model: flow_input.model }', 'a spread could supply any field'], + ['{ model: pickModel(flow_input.x) }', 'a call is not classifiable'], + ['flow_input.model + 1', 'not a bare reference'], + ['{ model: ', 'unparseable'] + ])('gives up on %s', (expr) => { + expect(parseProviderTransform({ type: 'javascript', expr } as any)).toBeUndefined() + }) +}) + +describe('agentModelWiringInputs', () => { + // The button writes `kind` only alongside a resource, since a provider is picked as a + // pair. Hiding a kind input it cannot write would leave the run without one. + it('keeps a kind input the model button cannot write', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ kind: flow_input.k, "resource": "$res:u/admin/claude", model: flow_input.m })`) + ]) + expect(agentModelWiringInputs(wiring)).toEqual(['m']) + }) + + it('hides a kind input it writes with the resource', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ kind: flow_input.k, resource: flow_input.r, model: flow_input.m })`) + ]) + expect(agentModelWiringInputs(wiring)?.sort()).toEqual(['k', 'm', 'r']) + }) + + // A thinking control is usable whatever the registry knows about the model — a ladder, a + // typed token, or why there is neither — so a wired effort is the button's there. + it('claims a wired reasoning_effort whatever the provider', () => { + const custom = resolveAgentModelWiring([ + agent( + `({ "kind": "customai", "resource": "$res:u/admin/custom", model: flow_input.m, reasoning_effort: flow_input.thinking })` + ) + ]) + expect(agentModelWiringInputs(custom)?.sort()).toEqual(['m', 'thinking']) + + const known = resolveAgentModelWiring([ + agent( + `({ "kind": "openai", "resource": "$res:u/admin/oai", "model": "gpt-4o", reasoning_effort: flow_input.thinking })` + ) + ]) + expect(agentModelWiringInputs(known)).toEqual(['thinking']) + }) + + // Agents on different fixed models leave the button no model to place the effort on, so + // it would say "Pick a model first" with nothing to pick. The modal keeps the input. + it('leaves a shared effort to the modal when the agents fix different models', () => { + const resource = `"kind": "anthropic", "resource": "$res:u/admin/claude"` + const wiring = resolveAgentModelWiring([ + agent(`({ ${resource}, "model": "claude-sonnet-5", reasoning_effort: flow_input.thinking })`), + agent(`({ ${resource}, "model": "claude-opus-5", reasoning_effort: flow_input.thinking })`) + ]) + expect(wiring?.fields.reasoning_effort).toBe('thinking') + expect(agentModelWiringInputs(wiring)).toEqual([]) + expect(showsModelButton(wiring)).toBe(false) + }) + + // The same one level up: agents on different providers leave the model menu nothing to + // list and no provider to gate a typed id, so the shared model input stays in the modal. + it('leaves a shared model to the modal when the agents fix different providers', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ "kind": "openai", "resource": "$res:u/admin/oai", model: flow_input.model })`), + agent( + `({ "kind": "azure_openai", "resource": "$res:u/admin/azure", model: flow_input.model })` + ) + ]) + expect(wiring?.fields.model).toBe('model') + expect(agentModelWiringInputs(wiring)).toEqual([]) + expect(showsModelButton(wiring)).toBe(false) + }) +}) + +// The modal is whatever this does not return, so the two cannot disagree about an input. +describe('composerOwnedInputs', () => { + const wiring = () => + resolveAgentModelWiring([ + agent(`({ "kind": "openai", "resource": "$res:u/admin/oai", model: flow_input.m })`) + ]) + + it('claims the model wiring and the attachments target together', () => { + expect(composerOwnedInputs(wiring(), { name: 'files' })?.sort()).toEqual(['files', 'm']) + }) + + it('claims the attachments input whatever the workspace can do with it', () => { + // No object storage is a state the paperclip shows, not a handover: the modal could + // not upload either, and the run would fail fetching a key typed there. + expect(composerOwnedInputs(undefined, { name: 'files' })).toEqual(['files']) + }) + + it('leaves an input no control fits to the modal', () => { + expect(composerOwnedInputs(wiring(), undefined)).toEqual(['m']) + }) +}) + +describe('resolveAgentModelWiring', () => { + const fixedResource = `"kind": "anthropic", "resource": "$res:u/admin/claude"` + + it('drives a field every agent reads from the same input', () => { + const wiring = resolveAgentModelWiring([ + agent( + `({ ${fixedResource}, "model": "claude-sonnet-5", reasoning_effort: flow_input.thinking })` + ), + agent( + `({ ${fixedResource}, "model": "claude-opus-5", reasoning_effort: flow_input.thinking })` + ) + ]) + expect(wiring?.fields).toEqual({ reasoning_effort: 'thinking' }) + // The agents run different models, so there is no single one to name. + expect(wiring?.fixed).toEqual({ kind: 'anthropic', resource: '$res:u/admin/claude' }) + }) + + it('drops a field the agents disagree about', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, reasoning_effort: flow_input.thinking })`), + agent(`({ ${fixedResource}, reasoning_effort: flow_input.other })`) + ]) + expect(wiring?.fields.reasoning_effort).toBeUndefined() + }) + + // A nested agent is the parent agent's tool, not a step the reader is talking to: the + // graph walks it as a child module, and counting it here would defeat the chat's own + // model control (this is the shape of the all-tools example flow). + it("ignores an agent carried as another agent's tool", () => { + const parent = agent('flow_input.model') + ;(parent.value as any).tools = [ + { + id: 'summarize', + value: { + tool_type: 'flowmodule', + type: 'aiagent', + tools: [], + input_transforms: { + provider: { type: 'static', value: { kind: 'anthropic', model: 'claude-sonnet-5' } } + } + } + } + ] + expect(resolveAgentModelWiring([parent])).toEqual({ + whole: 'model', + fields: {}, + fixed: {}, + someAgentCannotRun: false + }) + }) + + it('finds an agent inside a loop or a branch', () => { + const inner = agent(`({ ${fixedResource}, model: flow_input.model })`) + const loop = { + id: 'loop', + value: { type: 'forloopflow', modules: [inner] } + } as unknown as FlowModule + const branch = { + id: 'branch', + value: { type: 'branchone', default: [], branches: [{ modules: [inner] }] } + } as unknown as FlowModule + expect(resolveAgentModelWiring([loop])?.fields.model).toBe('model') + expect(resolveAgentModelWiring([branch])?.fields.model).toBe('model') + }) + + // The control writes one flow input; an agent that fixes the field instead never reads + // it, so offering the control would move one agent and leave the other where it was. + it('does not offer a field one agent wires and another fixes', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, model: flow_input.model })`), + agent(`({ ${fixedResource}, "model": "claude-opus-5" })`) + ]) + expect(wiring?.fields.model).toBeUndefined() + expect(wiring?.fixed.model).toBeUndefined() + }) + + // Disagreeing about the model is not the same as having no model: the flow runs, on a + // different one per agent, and the composer has nothing to fix. + it('says nothing about a model the agents merely disagree about', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, "model": "claude-sonnet-5" })`), + agent(`({ ${fixedResource}, "model": "claude-opus-5" })`) + ]) + expect(agentModelGap(wiring)).toBeUndefined() + }) + + it('still reports an agent with nothing to call', () => { + expect( + agentModelGap(resolveAgentModelWiring([agent(`({ "kind": "openai", "model": "" })`)])) + ).toBe('Pick a provider and model on the AI agent step to use this chat.') + }) + + // An expression the parser cannot account for could supply anything, so the agents it + // belongs to cannot be spoken for either. + it("offers nothing when one agent's provider cannot be read", () => { + expect( + resolveAgentModelWiring([ + agent(`({ ${fixedResource}, model: flow_input.model })`), + agent(`({ ...base, model: flow_input.model })`) + ]) + ).toBeUndefined() + }) + + // Disagreement is not the same as absence, but an agent with an empty model still + // cannot run, however well its neighbour is configured. + it('keeps warning when one agent has no model and another does', () => { + const wiring = resolveAgentModelWiring([ + agent(`({ ${fixedResource}, "model": "claude-sonnet-5" })`), + agent(`({ ${fixedResource}, "model": "" })`) + ]) + expect(agentModelGap(wiring)).toBe( + 'Pick a provider and model on the AI agent step to use this chat.' + ) + }) + + it('refuses a flow mixing whole-object and field-by-field wiring', () => { + expect( + resolveAgentModelWiring([ + agent('flow_input.provider'), + agent(`({ ${fixedResource}, model: flow_input.model })`) + ]) + ).toBeUndefined() + }) +}) + +describe('withoutRejectedEffort', () => { + const wiring = (fields: Record, fixed: Record = {}) => + ({ fields, fixed }) as any + + // The live 400 this guards: Anthropic turns any effort into adaptive thinking, which + // Haiku rejects outright ("adaptive thinking is not supported on this model"). + it('drops an effort the chosen model rejects', () => { + const values = { model: 'claude-haiku-4-5-20251001', reasoning_effort: 'high' } + expect( + withoutRejectedEffort( + wiring({ model: 'model', reasoning_effort: 'reasoning_effort' }, { kind: 'anthropic' }), + values + ) + ).toEqual({ model: 'claude-haiku-4-5-20251001', reasoning_effort: '' }) + }) + + it('keeps an effort the model takes', () => { + const values = { model: 'claude-sonnet-5', reasoning_effort: 'high' } + expect( + withoutRejectedEffort( + wiring({ model: 'model', reasoning_effort: 'reasoning_effort' }, { kind: 'anthropic' }), + values + ) + ).toBe(values) + }) + + // Clearing on a guess would override the author's own default. + it('leaves the value alone for a family the registry cannot speak for', () => { + const values = { model: 'some-model', reasoning_effort: 'high' } + expect( + withoutRejectedEffort( + wiring({ model: 'model', reasoning_effort: 'reasoning_effort' }, { kind: 'customai' }), + values + ) + ).toBe(values) + }) + + // A flow that wires `provider` as one object keeps the effort inside it, so reading + // `fields.reasoning_effort` finds nothing and the 400 would go out unchecked. + it('clears the effort inside a whole-object provider input', () => { + const wiring = resolveAgentModelWiring([agent('flow_input.provider')]) + const values = { + provider: { + kind: 'anthropic', + model: 'claude-haiku-4-5-20251001', + reasoning_effort: 'high' + } + } + expect(withoutRejectedEffort(wiring, values)).toEqual({ + provider: { + kind: 'anthropic', + model: 'claude-haiku-4-5-20251001', + reasoning_effort: '' + } + }) + }) + + it('leaves a whole-object provider alone when the model takes the effort', () => { + const wiring = resolveAgentModelWiring([agent('flow_input.provider')]) + const values = { + provider: { kind: 'anthropic', model: 'claude-sonnet-5', reasoning_effort: 'high' } + } + expect(withoutRejectedEffort(wiring, values)).toBe(values) + }) + + // A model that reasons can still refuse a particular token, and the provider answers with a + // 400 on every turn. + it('drops a level or off token a reasoning model does not take', () => { + const openai = wiring({ model: 'model', reasoning_effort: 'effort' }, { kind: 'openai' }) + expect(withoutRejectedEffort(openai, { model: 'gpt-5', effort: 'none' }).effort).toBe('') + expect(withoutRejectedEffort(openai, { model: 'gpt-5.1', effort: 'xhigh' }).effort).toBe('') + const kept = { model: 'gpt-5.1', effort: 'none' } + expect(withoutRejectedEffort(openai, kept)).toBe(kept) + }) +}) + +/** + * The shape agent chat is usually built in: one agent answers the reader, others do work of + * their own in branches. A sub-agent never sees the message, so what it runs on is not a + * setting this conversation has. + */ +describe('agents that do not read the message', () => { + const answerer = (provider: string) => + ({ + id: 'answerer', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + user_message: { type: 'javascript', expr: 'flow_input.user_message' }, + provider: { type: 'javascript', expr: provider } + } + } + }) as unknown as FlowModule + const subAgent = (provider: string, message = "'critique: ' + results.answerer") => + ({ + id: 'critic', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + user_message: { type: 'javascript', expr: message }, + provider: { type: 'javascript', expr: provider } + } + } + }) as unknown as FlowModule + + const wired = `({ kind: 'anthropic', resource: '$res:u/admin/c', model: flow_input.model })` + const fixed = `({ kind: 'anthropic', resource: '$res:u/admin/c', model: 'claude-sonnet-5' })` + + it('keeps the model control when only a sub-agent fixes its own model', () => { + const wiring = resolveAgentModelWiring([answerer(wired), subAgent(fixed)]) + expect(wiring?.fields.model).toBe('model') + }) + + // The worker runs a statement body too, so reading the message there is still reading it. + it('counts an agent that reads the message in a statement body', () => { + const wiring = resolveAgentModelWiring([ + subAgent(wired, 'const m = flow_input.user_message\nreturn m'), + subAgent(fixed) + ]) + expect(wiring?.fields.model).toBe('model') + }) + + // Two agents both answering the reader still have to agree: either might be the one + // that replies, so a control moving one of them would be a lie about the other. + it('still needs agreement among the agents that do read the message', () => { + const wiring = resolveAgentModelWiring([ + answerer(wired), + subAgent(fixed, 'flow_input.user_message') + ]) + expect(wiring?.fields.model).toBeUndefined() + }) + + // Nothing to scope to means the flow is shaped in some way this cannot read, so every + // agent counts again rather than none. + it('falls back to every agent when none reads the message', () => { + const wiring = resolveAgentModelWiring([subAgent(wired), subAgent(fixed)]) + expect(wiring?.fields.model).toBeUndefined() + }) + + // The editor writes the dot form, but an author may hand-edit either. A shape this does + // not recognise drops that agent out of the unanimity check it should be part of. + it.each([ + ["flow_input['user_message']", 'bracket access'], + ['flow_input?.user_message', 'optional chaining'], + ["'Answer politely: ' + flow_input.user_message", 'embedded in a prompt'], + ['flow_input.user_message + flow_input.tone', 'read alongside another input'], + ['flow_input.user_message // the message', 'a trailing comment'], + ['flow_input.user_message\n// why', 'a comment on its own last line'] + ])('treats %s as reading the message', (message) => { + const wiring = resolveAgentModelWiring([answerer(wired), subAgent(fixed, message as string)]) + expect(wiring?.fields.model).toBeUndefined() + }) + + it.each([ + ['// see flow_input.user_message', 'a mention in a comment'], + ["'flow_input.user_message'", 'a mention in a string'], + ['flow_input.user_message_extra', 'a different input with the same prefix'] + ])('does not treat %s as reading the message', (message) => { + const wiring = resolveAgentModelWiring([answerer(wired), subAgent(fixed, message as string)]) + expect(wiring?.fields.model).toBe('model') + }) +}) + +// An author annotating their own provider must not lose the model control for it: a +// comment beside the expression is not another expression. +describe('parseProviderTransform with comments', () => { + it.each([ + ['flow_input.provider // the one to use', 'a trailing line comment'], + ['flow_input.provider /* the one to use */', 'a trailing block comment'], + ['/* pick one */ flow_input.provider', 'a leading comment'] + ])('reads a whole-object reference despite %s', (expr) => { + expect(parseProviderTransform({ type: 'javascript', expr })?.whole).toBe('provider') + }) + + it('reads an object literal with a comment inside it', () => { + const wiring = parseProviderTransform({ + type: 'javascript', + expr: `({ kind: 'anthropic', /* fixed */ resource: '$res:u/admin/c', model: flow_input.model })` + }) + expect(wiring?.fields.model).toBe('model') + }) + + // The check exists to reject an expression with something else beside it; a second + // expression is still something else. + it('still refuses a second expression beside it', () => { + expect( + parseProviderTransform({ type: 'javascript', expr: 'flow_input.provider, 1' }) + ).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/agentChatInputs.ts b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts new file mode 100644 index 0000000000..0ebaa9d918 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/agentChatInputs.ts @@ -0,0 +1,380 @@ +import type { AIProvider, FlowModule, InputTransform } from '$lib/gen' +import { explicitOffToken, getReasoningCapability } from '$lib/components/copilot/reasoningRegistry' +import { carriedReasoning } from '$lib/components/copilot/chatModelSettings' +import { parseExpressionAt } from 'acorn' +import { agentSteps, flowInputReads } from './agentAttachmentInput' + +/** Block and line comments removed, so what is left is only what affects the value. */ +function withoutComments(source: string): string { + return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/[^\n]*/g, '') +} + +/** + * An expression wrapped so acorn will read it: parenthesised, because a leading `{` would + * otherwise parse as a block, and on its own line, because a trailing `// comment` would + * otherwise swallow the closing paren and make the whole thing unparseable. + */ +function parenthesised(expr: string): string { + return `(\n${expr}\n)` +} + +/** The flow input the server requires on a chat-enabled flow, and stores as the message. */ +const MESSAGE_INPUT = 'user_message' + +/** + * The agents the reader is talking to: the ones the chat's message is fed to. + * + * A flow commonly runs one agent on the message and others on work of their own — a + * critic reading `results.x`, a classifier in a branch. Those never see what was typed, + * so what they run on is not a setting this conversation has: letting one of them differ + * on the model would take the model control away from the agent that does answer. + * + * The message need not be the whole prompt — an author wraps it in context freely — so + * this asks whether the expression reads it at all. A flow where no agent reads it is one + * shaped in some way this cannot speak for, and every agent counts again rather than none. + */ +function chatFacingAgents(modules: FlowModule[] | undefined): FlowModule[] { + const agents = agentSteps(modules) + const facing = agents.filter((module) => { + const transform = (module.value as any).input_transforms?.[MESSAGE_INPUT] + return transform?.type === 'javascript' && readsFlowInput(transform.expr, MESSAGE_INPUT) + }) + return facing.length > 0 ? facing : agents +} + +/** Whether an expression reads `flow_input.`, as `flowInputReads` parses it. */ +function readsFlowInput(expr: string, name: string): boolean { + return flowInputReads(expr)?.has(name) ?? false +} + +/** A provider value as the agent stores it. */ +export type AgentModel = { kind?: string; model?: string; reasoning_effort?: string } + +/** The provider fields the composer can read or drive. */ +const PROVIDER_FIELDS = ['kind', 'resource', 'model', 'reasoning_effort'] as const +export type ProviderField = (typeof PROVIDER_FIELDS)[number] + +/** + * How an AI agent step's `provider` is supplied, field by field. + * + * The agent takes one `provider` object, so an author who wants the chat to choose only + * the model writes the rest as literals around it: + * + * { kind: 'anthropic', resource: '$res:u/admin/claude', model: flow_input.model } + * + * `fields` names the flow input behind each field the author exposed, `fixed` holds the + * literals, and `whole` covers the plain `flow_input.x` case where one input carries the + * entire object. Reading them apart is what lets the composer offer exactly the knobs the + * flow exposed — and stops a partial expression from being mistaken for a whole-object + * one, which would write a provider object into an input the flow reads as a model name. + */ +export type AgentModelWiring = { + whole?: string + fields: Partial> + fixed: Partial> + /** + * One of the agents names no resource or no model of its own and no flow input feeds + * it, so that agent's run fails whatever the others do. Held apart from the fields, + * which describe what the composer may offer. + */ + someAgentCannotRun?: boolean +} + +/** The flow input behind `flow_input.x`, `flow_input?.x` or `flow_input['x']`. */ +function flowInputName(node: any): string | undefined { + const member = node?.type === 'ChainExpression' ? node.expression : node + if (member?.type !== 'MemberExpression') return undefined + if (member.object?.type !== 'Identifier' || member.object.name !== 'flow_input') return undefined + if (!member.computed && member.property?.type === 'Identifier') return member.property.name + if (member.computed && member.property?.type === 'Literal') { + return typeof member.property.value === 'string' ? member.property.value : undefined + } + return undefined +} + +/** A property's name, for the plain `key:` and `'key':` forms only. */ +function propertyKey(property: any): string | undefined { + if (property?.type !== 'Property' || property.computed) return undefined + if (property.key?.type === 'Identifier') return property.key.name + if (property.key?.type === 'Literal' && typeof property.key.value === 'string') { + return property.key.value + } + return undefined +} + +/** + * Read a `provider` input transform. Anything this cannot account for in full returns + * undefined rather than a guess: the composer then leaves the field alone instead of + * writing into an expression it does not understand. + */ +export function parseProviderTransform( + transform: InputTransform | undefined +): AgentModelWiring | undefined { + if (transform?.type === 'static') { + const value = transform.value + if (!value || typeof value !== 'object') return undefined + const fixed: AgentModelWiring['fixed'] = {} + for (const field of PROVIDER_FIELDS) { + if (value[field] !== undefined) fixed[field] = value[field] + } + return { fields: {}, fixed } + } + if (transform?.type !== 'javascript') return undefined + + // The author's own text may already be wrapped, so any balanced surround is fine — what + // the span check rejects is an expression with something else beside it. + const source = parenthesised(transform.expr) + let node: any + try { + node = parseExpressionAt(source, 0, { ecmaVersion: 'latest' }) + } catch { + return undefined + } + // A comment beside the expression is not another expression: the author annotating their + // own provider must not cost them the model control. Dropped before the check so what + // remains is only what would change the value — and so a paren inside a comment is not + // counted as one of the wrapping pair. + const before = withoutComments(source.slice(0, node.start)) + const after = withoutComments(source.slice(node.end)) + if (!/^[\s(]*$/.test(before) || !/^[\s)]*$/.test(after)) return undefined + if ((before.match(/\(/g)?.length ?? 0) !== (after.match(/\)/g)?.length ?? 0)) return undefined + + const whole = flowInputName(node) + if (whole) return { whole, fields: {}, fixed: {} } + if (node.type !== 'ObjectExpression') return undefined + + const fields: AgentModelWiring['fields'] = {} + const fixed: AgentModelWiring['fixed'] = {} + for (const property of node.properties) { + const key = propertyKey(property) + // A spread or a computed key could supply any field, so nothing here is knowable. + if (!key) return undefined + if (!(PROVIDER_FIELDS as readonly string[]).includes(key)) continue + const name = flowInputName(property.value) + if (name) { + fields[key as ProviderField] = name + } else if (property.value?.type === 'Literal') { + fixed[key as ProviderField] = property.value.value + } else { + return undefined + } + } + return { fields, fixed } +} + +/** How one agent supplies a provider field: from an input, as a literal, or not at all. */ +type FieldSupply = + | { kind: 'wired'; name: string } + | { kind: 'fixed'; value: any } + | { kind: 'absent' } + +/** Whether one agent supplies a field with nothing usable: no input, and no literal. */ +function agentFieldEmpty(wiring: AgentModelWiring, field: ProviderField): boolean { + if (wiring.fields[field] !== undefined) return false + const value = wiring.fixed[field] + return value === undefined || value === '' +} + +function fieldSupply(wiring: AgentModelWiring, field: ProviderField): FieldSupply { + const name = wiring.fields[field] + if (name !== undefined) return { kind: 'wired', name } + const value = wiring.fixed[field] + if (value !== undefined) return { kind: 'fixed', value } + return { kind: 'absent' } +} + +/** + * The provider wiring the chat can act on, across every AI agent in the flow. + * + * With several agents a field is drivable when they agree on it: one flow input feeding + * it, or one literal fixing it. Where they disagree there is no single value to show or + * write, so that field is dropped and the others still work. A flow mixing whole-object + * and field-by-field wiring is ambiguous throughout and yields nothing. + */ +export function resolveAgentModelWiring( + modules: FlowModule[] | undefined +): AgentModelWiring | undefined { + const agents = chatFacingAgents(modules) + const parsed = agents.map((agent) => + parseProviderTransform((agent.value as any).input_transforms?.['provider']) + ) + if (parsed.length === 0) return undefined + // An agent whose provider cannot be read is an agent the composer cannot speak for: + // dropping it would let the rest declare a control that governs only some of them. + if (parsed.some((wiring) => wiring === undefined)) return undefined + const wirings = parsed as AgentModelWiring[] + // Whether any single agent has nothing to call, which stays true however the others + // are wired — the gap message is about that agent, not about their agreement. + const someAgentCannotRun = wirings.some( + (wiring) => + !wiring.whole && (agentFieldEmpty(wiring, 'resource') || agentFieldEmpty(wiring, 'model')) + ) + if (wirings.length === 1) return { ...wirings[0], someAgentCannotRun } + + const wholes = new Set(wirings.map((w) => w.whole)) + if (wholes.size === 1 && !wholes.has(undefined)) { + return { whole: [...wholes][0], fields: {}, fixed: {}, someAgentCannotRun } + } + if (wirings.some((w) => w.whole !== undefined)) return undefined + + const fields: AgentModelWiring['fields'] = {} + const fixed: AgentModelWiring['fixed'] = {} + for (const field of PROVIDER_FIELDS) { + // Every agent has to supply the field the same way for the composer to speak for + // them all. One wired name among agents that otherwise fix it is not agreement: + // the control would move that one agent and leave the others where they are. + const supplies = new Set(wirings.map((w) => JSON.stringify(fieldSupply(w, field)))) + // Disagreement leaves the field neither editable nor known: a control offered here + // would govern one agent while the rest ran on something else. + if (supplies.size > 1) continue + const supply: FieldSupply = JSON.parse([...supplies][0]) + if (supply.kind === 'wired') fields[field] = supply.name + else if (supply.kind === 'fixed') fixed[field] = supply.value + } + return { fields, fixed, someAgentCannotRun } +} + +/** + * Why the chat cannot run, when the agent's own provider is incomplete. + * + * A freshly added agent carries `{ kind: 'openai', model: '', resource: '' }`, so it names + * a provider kind while having nothing to call — the run fails and the chat can do nothing + * about it, because no flow input feeds either field. Saying so beats a dead model button. + * A field the flow exposes is never a gap: the reader picks it in the composer. + */ +export function agentModelGap(wiring: AgentModelWiring | undefined): string | undefined { + // No agent, several of them, or an expression we cannot read: not ours to judge. + if (!wiring || wiring.whole) return undefined + // Asked of each agent rather than of what they agree on: agents that merely disagree + // about the model all have one, and the message would be false — while an agent with + // an empty model still cannot run, however well the others are configured. + return wiring.someAgentCannotRun + ? 'Pick a provider and model on the AI agent step to use this chat.' + : undefined +} + +/** + * The provider fields the model button edits. Everything else wired to an input is left to + * the Configure-inputs modal, and the button draws no control for it. + * + * A field is the button's only where its control can be used, which each field makes depend + * on the one before it. An input promoted without a usable control is one nothing can edit. + * - `resource` is always usable: the submenu lists the workspace's AI resources. + * - `kind` is written only alongside a resource, since a provider is picked as a pair. Wired + * with the resource fixed, the button has nothing to write it with. + * - `model` needs a provider to list models for and to gate the typed entry: a wired resource + * or kind, or one fixed kind. Agents that fix different kinds leave none. + * - `reasoning_effort` needs a model to place the effort on: a driven model or one fixed + * model. Agents that fix different models leave none. + */ +export function composerDrivenFields(wiring: AgentModelWiring): Set { + if (wiring.whole) return new Set(PROVIDER_FIELDS) + const wired = (field: ProviderField) => wiring.fields[field] !== undefined + const driven = new Set() + if (wired('resource')) { + driven.add('resource') + if (wired('kind')) driven.add('kind') + } + const providerKnown = wired('resource') || wired('kind') || fixedOne(wiring, 'kind') + if (wired('model') && providerKnown) driven.add('model') + const modelKnown = driven.has('model') || fixedOne(wiring, 'model') + if (wired('reasoning_effort') && modelKnown) driven.add('reasoning_effort') + return driven +} + +/** Whether every agent fixes the field to the same non-empty literal. */ +function fixedOne(wiring: AgentModelWiring, field: ProviderField): boolean { + const value = wiring.fixed[field] + return value !== undefined && value !== '' +} + +/** The flow inputs the model button writes, so the modal does not ask for them twice. */ +export function agentModelWiringInputs(wiring: AgentModelWiring | undefined): string[] { + if (!wiring) return [] + if (wiring.whole) return [wiring.whole] + return [...composerDrivenFields(wiring)] + .map((field) => wiring.fields[field]) + .filter((name): name is string => !!name) +} + +/** + * Whether the composer draws a model button at all: something to write, or a fixed model to + * name. Agents that fix different models leave neither. + */ +export function showsModelButton(wiring: AgentModelWiring | undefined): boolean { + if (!wiring) return false + return agentModelWiringInputs(wiring).length > 0 || fixedOne(wiring, 'model') +} + +/** + * The flow inputs the composer edits, and therefore the ones the Configure-inputs modal must + * not ask for. The modal is whatever is left, so this is the single answer to "who edits + * this" rather than a list kept in step with the controls that render. + * + * Only the shape of the flow decides it. Whether a control can act *right now* — no rules + * for a provider's thinking levels, say — is a state that control shows, not a reason to + * hand the input to an editor that would be no more able. The attachments target is the + * composer's other owner: the input its paperclip uploads into, where the flow has one. + */ +export function composerOwnedInputs( + wiring: AgentModelWiring | undefined, + attachmentsTarget: { name: string } | undefined +): string[] { + return [...agentModelWiringInputs(wiring), ...(attachmentsTarget ? [attachmentsTarget.name] : [])] +} + +/** + * The run's inputs with a reasoning effort the chosen model cannot take removed. + * + * The model button reconciles the two when the reader switches model, which covers the only + * way the composer can put them out of step. It is not the only way they get out of step: + * a value stored from an earlier visit, a default the flow author wrote, or a model chosen + * before the effort was, all arrive already mismatched — and the provider answers a + * mismatch with a 400 that names neither input ("adaptive thinking is not supported on this + * model"). Checked here, where the run's arguments are settled, so every route is covered. + * + * Only where the registry positively knows the model rejects it. An unknown family keeps + * whatever the author wrote: dropping a value on a guess would override their own default. + */ +export function withoutRejectedEffort( + wiring: AgentModelWiring | undefined, + values: Record +): Record { + if (!wiring) return values + + // One input carrying the whole provider object: the three fields are read from it and + // the effort is cleared inside it, since that is where the agent will look for them. + if (wiring.whole) { + const provider = values[wiring.whole] + if (!provider || typeof provider !== 'object') return values + if (!rejectsEffort(provider.kind, provider.model, provider.reasoning_effort)) return values + return { ...values, [wiring.whole]: { ...provider, reasoning_effort: '' } } + } + + const effortInput = wiring.fields.reasoning_effort + if (!effortInput) return values + const kindInput = wiring.fields.kind + const modelInput = wiring.fields.model + const rejected = rejectsEffort( + kindInput ? values[kindInput] : wiring.fixed.kind, + modelInput ? values[modelInput] : wiring.fixed.model, + values[effortInput] + ) + return rejected ? { ...values, [effortInput]: '' } : values +} + +/** + * Whether the registry positively says this model will not take this effort: no reasoning at + * all, a level it does not have (`xhigh` on `gpt-5.1`), or an off token it cannot honour + * (`none` on `gpt-5`). The same rule the button applies when the model changes. + */ +function rejectsEffort(provider: unknown, model: unknown, effort: unknown): boolean { + if (typeof effort !== 'string' || effort === '') return false + if (typeof provider !== 'string' || typeof model !== 'string' || !provider || !model) { + return false + } + const capability = getReasoningCapability(provider as AIProvider, model) + if (!capability.known) return false + const offToken = explicitOffToken(provider as AIProvider, model) + return carriedReasoning(effort, offToken, capability) === undefined +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatProps.ts b/frontend/src/lib/components/flows/conversations/flowChatProps.ts new file mode 100644 index 0000000000..73500b31bd --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatProps.ts @@ -0,0 +1,19 @@ +/** What separates the chat from what sits above it. `boxed` is its own panel, corners + * clipped so the sidebar's edge follows them; `top` a dividing line under an enclosing + * header; `none` for a surface where the chat is the whole pane. */ +export type ChatFrame = 'boxed' | 'top' | 'none' + +export const FRAME_CLASS: Record = { + boxed: 'border rounded-md', + top: 'border-t', + none: '' +} + +/** + * Which flow a chat is for, as something that holds still: `identity` where a surface has + * one, since `path` follows the path field as its author types. `||`, not `??`: an empty + * identity is no identity. + */ +export function chatFlowKey(props: { path: string; identity?: string }): string { + return props.identity || props.path +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts new file mode 100644 index 0000000000..6ef18b7212 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -0,0 +1,695 @@ +import type { AttachmentUpload, Chat, ChatMessage, ChatState } from 'windmill-chat' +import type { + ChatSendRequestOptions, + ChatViewHost +} from '$lib/components/copilot/chat/chatViewHost' +import type { DisplayMessage } from '$lib/components/copilot/chat/shared' +import type { AIAutonomyMode } from '$lib/components/copilot/chat/AIChatManager.svelte' +import { isPlanCardTool } from '$lib/components/copilot/chat/planMode' +import { AttachedFilesStore } from '$lib/components/copilot/chat/files/attachedFiles.svelte' +import { SessionArtifactsStore } from '$lib/components/copilot/chat/artifacts/artifactsState.svelte' +import type { AttachedBlob } from '$lib/components/copilot/chat/blobUtils' +import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils' +import type { AttachedTextFile } from '$lib/components/copilot/chat/textFileUtils' +import { + prefersInstantReveal, + TypewriterReveal, + type TypewriterRevealOptions +} from '$lib/components/copilot/chat/typewriterReveal' +import { JobService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { attachmentLanes } from './messageAttachments' +import type { AttachmentsTarget } from './agentAttachmentInput' + +export type FlowChatViewHostOptions = { + /** The flow inputs sent next to `user_message` with every turn. */ + additionalInputs?: () => Record | undefined + /** The flow input the composer's attachments feed. Undefined where the flow has none: + * the chat then takes no attachments at all. */ + attachmentsTarget?: () => AttachmentsTarget | undefined + /** Why attaching is off despite the flow taking attachments — no object storage, say. + * Undefined while the workspace has not answered: an explanation must not be a guess. */ + attachmentsUnavailable?: () => string | undefined + /** The workspace the transcript's paths resolve against, and a retry reads its run from. */ + workspace?: () => string | undefined + /** Whether sending is refused right now (a deployment in progress, say). The composer + * is disabled on the same condition; this covers the sends the composer does not + * make itself: a queued message going out, a retry. */ + sendDisabled?: () => boolean + /** Flow inputs edited by a control beside the composer, such as the model button. A retry + * takes their current value rather than the failed turn's. */ + inputsShownInComposer?: () => string[] + /** Injectables for tests: the clock and scheduler behind the typewriter pacing. */ + revealOptions?: Pick +} + +function isBusy(status: ChatState['status']): boolean { + return status === 'submitted' || status === 'streaming' +} + +/** The chat's own signal for a send stopped before it ran; nothing to tell the reader. */ +function isAbort(e: unknown): boolean { + return e instanceof Error && e.name === 'AbortError' +} + +type Queue = { text: string; images: AttachedImage[]; blobs: AttachedBlob[] } + +function emptyQueue(): Queue { + return { text: '', images: [], blobs: [] } +} + +/** A tool's arguments or result as the card shows them: parsed where the string is JSON. */ +function parseToolPayload(raw: string | undefined): unknown { + if (raw === undefined || raw === '') return undefined + try { + return JSON.parse(raw) + } catch { + return raw + } +} + +/** + * Whether the turn the user message at `index` started failed: its last row before the + * next user message reports `success: false`. The last row, not any row: a tool call can + * fail and the agent still answer, and that turn completed. + */ +export function turnFailed(messages: readonly ChatMessage[], index: number): boolean { + let last: ChatMessage | undefined + for (let i = index + 1; i < messages.length; i++) { + const message = messages[i] + if (message.role === 'user') break + last = message + } + return last?.success === false +} + +/** Whether the latest turn failed, per `turnFailed`. False before any turn. */ +function lastTurnFailed(messages: readonly ChatMessage[]): boolean { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === 'user') return turnFailed(messages, i) + } + return false +} + +/** + * The step name says which AI agent step wrote a message, so it only tells the reader + * anything once the conversation holds more than one. Counted over the transcript rather + * than over the flow's steps: a conversation outlives edits to the flow, so it can carry + * labels from a shape the flow no longer has. + */ +export function showsStepNames(messages: readonly ChatMessage[]): boolean { + return new Set(messages.map((m) => m.stepName).filter(Boolean)).size > 1 +} + +/** How much of a streaming message is on screen, per lane, in characters. */ +export type Revealed = { content: number; reasoning: number } + +/** What a display row can only learn beyond the message itself. Every lookup is optional. */ +export type DisplayLookups = { + /** The workspace an attachment's download link points into. */ + workspace?: string + /** How much of a pending assistant row the pacing has put on screen. */ + revealed?: (message: ChatMessage) => Revealed | undefined +} + +/** + * What a failed tool call returned, as the card's error: the card shows the error in place of + * the result. A job failure is stored as `{ message, name, stack }`; an MCP failure as plain text. + */ +function toolErrorText(result: unknown): string | undefined { + if (typeof result === 'string') return result || undefined + if (result && typeof result === 'object') { + const message = (result as { message?: unknown }).message + return typeof message === 'string' ? message : JSON.stringify(result, null, 2) + } + return undefined +} + +/** The name the worker gives the structured-output tool: suffixed when an agent tool already has it. */ +const STRUCTURED_OUTPUT_CALL = /^structured_output(_\d+)?$/ + +/** + * `busy`: the latest turn is still running, so a failed tool call is not yet its outcome. + * `stopped`: ids of the user messages of the turns the reader stopped, as message id or row id. + * A stopped turn never offers Retry, though Stop leaves its failed tool row or the cancelled + * flow's failure as its last message. + */ +export function toDisplayMessages( + messages: readonly ChatMessage[], + busy = false, + stopped: ReadonlySet = new Set(), + lookups: DisplayLookups = {} +): DisplayMessage[] { + let userIndex = 0 + const stepNames = showsStepNames(messages) + let latestUser = -1 + for (let i = messages.length - 1; i >= 0 && latestUser < 0; i--) { + if (messages[i].role === 'user') latestUser = i + } + return messages.flatMap((message, i): DisplayMessage[] => { + switch (message.role) { + case 'user': { + const index = userIndex++ + const settled = + !(busy && i === latestUser) && + !stopped.has(message.id) && + !(message.serverId && stopped.has(message.serverId)) + const { images, contextElements } = attachmentLanes(lookups.workspace, message.attachments) + return [ + { + role: 'user', + index, + content: message.content, + // Drives the shared Retry button. + error: (settled && turnFailed(messages, i)) || undefined, + images: images.length > 0 ? images : undefined, + contextElements: contextElements.length > 0 ? contextElements : undefined + } + ] + } + case 'tool': { + // The model's call and what the tool sent back, as the stream carried them or the + // worker stored them on the row. A row stored without them shows its name and job. + const toolName = message.tool?.name + const parameters = parseToolPayload(message.tool?.arguments) + const result = parseToolPayload(message.tool?.result) + const failed = message.success === false + // A call the turn finished without, stopped or lost before its row was written. The + // call a structured answer streams as never gets a result of its own, the answer + // being the turn's text, so it is not one. + const unfinished = + message.tool && + !message.pending && + !message.content && + !STRUCTURED_OUTPUT_CALL.test(message.tool.name) + ? `${message.tool.name} did not finish` + : undefined + const call: DisplayMessage = { + role: 'tool', + tool_call_id: message.id, + // The card's header is the row's text, which the server only words once the + // tool has returned; until then the row says what is running. + content: message.content || unfinished || (toolName ? `Running ${toolName}` : ''), + // Withheld for the copilot's two plan-mode names: `toolName` is what makes + // ToolExecutionDisplay render a plan card, and an agent tool that happened to + // share one would silently become one. + toolName: isPlanCardTool(toolName) ? undefined : toolName, + parameters, + result, + showDetails: parameters !== undefined || result !== undefined, + error: failed ? (toolErrorText(result) ?? message.content) : unfinished, + isLoading: message.pending && message.tool?.status === 'running', + jobId: message.jobId + } + // The tool card has no thinking section: the thinking that led to the call reads + // as a card of its own, just before it. + return message.reasoning + ? [ + { + role: 'assistant', + content: '', + reasoning: message.reasoning, + stepName: stepNames ? message.stepName : undefined, + jobId: message.jobId, + createdAt: message.createdAt + }, + call + ] + : [call] + } + default: { + // The pacing's prefix while the message streams; the whole text once it has + // settled, or the turn was stopped. + const revealed = message.pending ? lookups.revealed?.(message) : undefined + return [ + { + role: 'assistant', + content: revealed ? message.content.slice(0, revealed.content) : message.content, + // Only the message a turn is still writing: a finalized reasoning-only + // message must not look in progress. + streaming: message.pending || undefined, + reasoning: revealed + ? message.reasoning?.slice(0, revealed.reasoning) + : message.reasoning, + stepName: stepNames ? message.stepName : undefined, + jobId: message.jobId, + createdAt: message.createdAt + } + ] + } + } + }) +} + +/** The two paced lanes of one streaming message, and how much of each has been fed in. */ +type RevealLanes = { + content: TypewriterReveal + reasoning: TypewriterReveal + fed: Revealed +} + +/** + * Renders a flow run's conversation, as the `windmill-chat` SDK keeps it, through the + * copilot's chat components. The turn is a flow job rather than an LLM call this host + * makes, so what it can offer is what the SDK's `Chat` can: a message in, an answer + * streamed back, Stop. Every copilot-only field is answered with "no" (see ChatViewHost). + * + * The host owns its subscription to the chat, so a panel swapping chats mounts a new one. + */ +export class FlowChatViewHost implements ChatViewHost { + #chat: Chat + #options: FlowChatViewHostOptions + #state = $state.raw() as ChatState + #unsubscribe: () => void + + constructor(chat: Chat, options: FlowChatViewHostOptions = {}) { + this.#chat = chat + this.#options = options + this.#state = chat.getState() + this.#unsubscribe = chat.subscribe((state) => this.#onState(state)) + } + + #disposed = false + /** Stops following the chat, and drops what was queued: a flush still waiting on the + * turn's release would otherwise start a run from a panel that is gone. A send still + * uploading its attachments stops with the chat, which is the caller's to destroy; its + * draft is not handed back, since the composer it came from is gone too. */ + dispose() { + this.#disposed = true + this.#queue = emptyQueue() + this.#unsubscribe() + for (const id of Object.keys(this.#reveals)) this.#dropReveal(id) + } + + /** The latest `ChatState`, for what the interface reads beyond the seam (paging, loading). */ + get state(): ChatState { + return this.#state + } + + #onState(state: ChatState) { + const previous = this.#state + this.#state = state + this.#paceReveals(state.messages) + const landed = state.messages.filter( + (m) => m.serverId && this.#stoppedTurns.has(m.id) && !this.#stoppedTurns.has(m.serverId) + ) + if (landed.length > 0) { + this.#stoppedTurns = new Set([...this.#stoppedTurns, ...landed.map((m) => m.serverId!)]) + } + if (previous.conversationId !== state.conversationId) { + // A conversation opens at its end, whatever the reader was doing in the last one. + this.#automaticScroll = true + // The queue was typed into the conversation that just went away; a message sent + // after the switch would ride out of the wrong one, so it goes back to the composer. + this.dequeueMessage() + return + } + if (!isBusy(previous.status) && isBusy(state.status)) this.#turnsStarted++ + if (isBusy(previous.status) && !isBusy(state.status)) { + // The turn settled. What was typed during it goes out once the turn is released, + // not now: the chat publishes `idle` from inside its own `sendMessage`, which still + // counts the turn as open until it returns, and a send made before that would be + // refused as a second turn. After a failure it goes back to the composer instead, + // where the reader would rather look at the error than pile on. A failed flow + // settles as `idle` too, with its error as the answer, so the messages decide. + const succeeded = state.status === 'idle' && !lastTurnFailed(state.messages) + if (succeeded) void this.#turnDone.then(this.flushQueuedMessage) + else this.dequeueMessage() + } + } + + // Smooth streaming. The chat appends each delta to the pending assistant message as it + // arrives, in the coarse bursts the provider sends; what is shown is a prefix that a + // typewriter advances per lane, so the bursts read as continuous typing. Plain fields + // for the pacers, reactive counts for what they have put on screen. + #reveals: Record = {} + #revealed = $state>({}) + + #paceReveals(messages: readonly ChatMessage[]) { + const pending = new Set() + for (let i = 0; i < messages.length; i++) { + const message = messages[i] + if (message.role !== 'assistant' || !message.pending) continue + pending.add(message.id) + const lanes = (this.#reveals[message.id] ??= this.#newLanes(message.id)) + const content = message.content.slice(lanes.fed.content) + const reasoning = (message.reasoning ?? '').slice(lanes.fed.reasoning) + lanes.fed = { content: message.content.length, reasoning: (message.reasoning ?? '').length } + lanes.content.push(content) + lanes.reasoning.push(reasoning) + // A row the turn has moved past — a tool card now follows it — is shown whole: + // what the pacing still holds belongs above that card, not trickling in under it. + if (i < messages.length - 1) { + lanes.content.flush() + lanes.reasoning.flush() + } + } + for (const id of Object.keys(this.#reveals)) { + if (!pending.has(id)) this.#dropReveal(id) + } + } + + #newLanes(id: string): RevealLanes { + const options = this.#options.revealOptions ?? {} + const instant = options.instant ?? prefersInstantReveal() + const lane = (kind: keyof Revealed) => + new TypewriterReveal({ + ...options, + instant, + onReveal: (chunk) => { + const current = this.#revealed[id] ?? { content: 0, reasoning: 0 } + this.#revealed[id] = { ...current, [kind]: current[kind] + chunk.length } + } + }) + this.#revealed[id] = { content: 0, reasoning: 0 } + return { + content: lane('content'), + reasoning: lane('reasoning'), + fed: { content: 0, reasoning: 0 } + } + } + + #dropReveal(id: string) { + this.#reveals[id]?.content.reset() + this.#reveals[id]?.reasoning.reset() + delete this.#reveals[id] + delete this.#revealed[id] + } + + // Transcript + displayMessages = $derived.by(() => + toDisplayMessages(this.#state.messages, isBusy(this.#state.status), this.#stoppedTurns, { + workspace: this.#options.workspace?.(), + revealed: (message) => this.#revealed[message.id] + }) + ) + /** The user message of each turn stopped in this view, by message id and, once the chat has + * read its row, row id: a reopened conversation or an older page rebuilds messages from rows, + * so a turn left before its row was read is not recognised there. Only this session knows; + * a reload shows the stopped turn as its rows left it. */ + #stoppedTurns = $state.raw>(new Set()) + get messages(): readonly unknown[] { + return this.#state.messages + } + contextTokens = 0 + get operatingWorkspace(): string | undefined { + return this.#options.workspace?.() + } + get loading(): boolean { + return isBusy(this.#state.status) + } + // No transcript restore to wait for: a flow chat renders from its rows. + sendPending = false + runHeldElsewhere = false + loadingLabel = undefined + compacting = false + // The answer streams into the message list itself, so the live lanes stay empty. + currentReply = '' + currentReasoning = '' + currentReasoningActive = false + reasoningHiddenIndicatorLabel = undefined + #automaticScroll = $state(true) + get automaticScroll(): boolean { + return this.#automaticScroll + } + enableAutomaticScroll = () => { + this.#automaticScroll = true + } + disableAutomaticScroll = () => { + this.#automaticScroll = false + } + + // Composer + instructions = '' + // The user message lands in the transcript before `sendMessage` awaits anything. + sendInFlight = false + /** + * `replayInputs` are a failed turn's own run arguments, read back from its job. They + * stand in for the composer's current inputs, so a retry runs the turn that failed + * rather than a new one wearing its text. + */ + sendRequest = async ( + options: ChatSendRequestOptions = {}, + replayInputs?: Record + ): Promise => { + const text = options.instructions?.trim() ?? '' + let images = options.images ?? [] + let blobs = options.blobs ?? [] + // The composer refuses an attachment-only send (requiresMessageText), so this is + // the same rule at the other end: nothing runs without a message. + if (!text) return false + if (this.loading) { + this.queueMessage(text, images, undefined, undefined, blobs) + return true + } + if (this.#options.sendDisabled?.()) { + // Refused, not dropped: the draft waits in the composer for sending to reopen. + this.#aiChatInput?.prependText(text, images, [], blobs) + return false + } + const target = this.#options.attachmentsTarget?.() + // The inputs modal does not ask for this input, so a required one is enforced here. A + // replay carries the files its run already has, in `replayInputs`, and attaches none. + if (!replayInputs && target?.required && images.length === 0 && blobs.length === 0) { + sendUserToast('This chat needs a file with each message. Attach one to send.', true) + this.#aiChatInput?.prependText(text, images, [], blobs) + return false + } + // A replay sends the arguments its run had, attachment references included. + const inputs = replayInputs ?? { ...(this.#options.additionalInputs?.() ?? {}) } + // The attachments are this input's only editor: a value stored for it in the inputs + // modal would otherwise ride along on every message. + if (target && !replayInputs) delete inputs[target.name] + // The composer caps files as they are attached, but a queue merged over several turns + // arrives here as one send, and the chat refuses more than a single-file input holds. + const cap = this.maxMessageAttachments + if (cap !== undefined && images.length + blobs.length > cap) { + const dropped = images.length + blobs.length - cap + images = images.slice(0, cap) + blobs = blobs.slice(0, Math.max(0, cap - images.length)) + sendUserToast( + cap === 1 + ? `This chat sends one attachment per message; ${dropped} file(s) were not sent.` + : `This chat sends up to ${cap} attachments per message; ${dropped} file(s) were not sent.`, + true + ) + } + const attachments: AttachmentUpload[] = target + ? [...images, ...blobs].map((attachment, index) => ({ + name: attachment.name ?? `attachment-${index + 1}`, + data: attachment.dataUrl, + mediaType: attachment.mediaType + })) + : [] + this.#automaticScroll = true + // A run that fails is reported through the chat's `onError` and as a failed message; + // the promise itself only rejects when the chat refuses the turn outright — a turn + // already running, an upload that failed, Stop pressed while it ran — and the draft + // is then handed back rather than dropped. The composer took it before calling, so + // nothing else would. + const turn = this.#chat + .sendMessage(text, { + inputs: replayInputs ?? (this.#options.additionalInputs?.() ? inputs : undefined), + attachments, + attachmentsInput: target + }) + .catch((e) => { + if (this.#disposed) return + if (attachments.length > 0 && !isAbort(e)) { + sendUserToast( + `Could not upload the attachments: ${e instanceof Error ? e.message : String(e)}`, + true + ) + } + // What was queued behind it comes back too, after it: the chat publishes `idle` + // when it withdraws the turn, and a queue left in place would be flushed as if + // the turn had run. + this.dequeueMessage() + this.#aiChatInput?.prependText(text, images, [], blobs) + }) + this.#turnDone = turn + await turn + return true + } + /** Settles when the chat has released the last turn this host started. */ + #turnDone: Promise = Promise.resolve() + cancel = () => { + // Stop means stop: what was typed during the run goes back to the composer rather + // than waiting there to go out after some later turn settles. + this.dequeueMessage() + const { messages, status } = this.#state + const turn = isBusy(status) ? [...messages].reverse().find((m) => m.role === 'user') : undefined + if (turn) { + this.#stoppedTurns = new Set( + [...this.#stoppedTurns, turn.id, turn.serverId].filter((id) => id !== undefined) + ) + } + void this.#chat.stop() + } + // Typed off the interface: a Svelte component's own type resolves differently + // across import specifiers, and the two would then not be assignable. + #aiChatInput: Parameters[0] = null + setAiChatInput: ChatViewHost['setAiChatInput'] = (aiChatInput) => { + this.#aiChatInput = aiChatInput + } + + // One message typed while the turn runs, sent whole with its attachments once the turn + // settles. Enter again appends a line rather than replacing what waits. + #queue = $state(emptyQueue()) + get queuedMessage(): string { + return this.#queue.text + } + queuedContext = undefined + get queuedImages(): AttachedImage[] { + return this.#queue.images + } + queuedFiles: AttachedTextFile[] = [] + get queuedBlobs(): AttachedBlob[] { + return this.#queue.blobs + } + queueMessage = ( + text: string, + images: AttachedImage[] = [], + _context?: unknown, + _files?: unknown, + blobs: AttachedBlob[] = [] + ) => { + const trimmed = text.trim() + if (!trimmed && images.length === 0 && blobs.length === 0) return + const queue = this.#queue + this.#queue = { + text: !trimmed ? queue.text : queue.text ? `${queue.text}\n${trimmed}` : trimmed, + images: [...queue.images, ...images], + blobs: [...queue.blobs, ...blobs] + } + } + /** Put the queued draft back in the composer, attachments included. */ + dequeueMessage = () => { + const { text, images, blobs } = this.#takeQueue() + if (!text && images.length === 0 && blobs.length === 0) return + this.#aiChatInput?.prependText(text, images, [], blobs) + } + flushQueuedMessage = () => { + // Same rule as sendRequest, read before the queue is drained: a turn with no message + // cannot run, and taking the queue for it would drop the attachments on the floor. + if (!this.#queue.text || this.#disposed) return + const { text, images, blobs } = this.#takeQueue() + void this.sendRequest({ instructions: text, images, blobs }) + } + #takeQueue(): Queue { + const taken = this.#queue + this.#queue = emptyQueue() + return taken + } + setComposerStaged = () => {} + setComposerHasDraft = () => {} + clearComposerStaged = () => {} + attachmentBytesExcluding = () => 0 + + // Per-message actions + storedImages = () => undefined + /** A retry reading back the turn it is about to replay. Deliberately not part of + * `loading`, which renders Stop: there is no run yet to stop. */ + #readingReplayArgs = false + /** Turns this chat has started, so a retry can tell whether one ran while it read the + * arguments, whether or not it is still running. */ + #turnsStarted = 0 + /** + * Run the turn at this position again with the arguments its job ran with, not the + * composer's current inputs (`inputsShownInComposer` aside). The position is in + * `displayMessages`, which holds more entries than the chat's messages. A purged job falls + * back to the current inputs. + */ + retryRequest = async (messageIndex: number) => { + const shown = this.displayMessages[messageIndex] + if (!shown || shown.role !== 'user' || this.loading || this.#readingReplayArgs) return + const message = this.#state.messages.filter((m) => m.role === 'user')[shown.index] + if (!message) return + const conversationId = this.#state.conversationId + const turnsStarted = this.#turnsStarted + const workspace = this.#options.workspace?.() + let replayInputs: Record | undefined + if (message.jobId && workspace) { + this.#readingReplayArgs = true + try { + const original = (await JobService.getJobArgs({ workspace, id: message.jobId })) as + | Record + | undefined + // `user_message` is the message itself, passed as the instructions below. + const { user_message: _sent, ...rest } = original ?? {} + const current = this.#options.additionalInputs?.() ?? {} + for (const name of this.#options.inputsShownInComposer?.() ?? []) { + if (name in current) rest[name] = current[name] + else delete rest[name] + } + replayInputs = rest + } catch (error) { + // Only a job that is gone justifies running with other inputs; after a blip or + // a 500 that would silently run a different turn. + if ((error as { status?: number })?.status !== 404) { + sendUserToast('Could not read what that turn ran with. Try again.', true) + return + } + } finally { + this.#readingReplayArgs = false + } + } + // The reader may have moved on while the arguments were read: to another conversation, + // where this turn does not belong, or by sending. Any turn started since counts, settled + // or not: they wrote the chat's latest message, and running this one now would answer + // something they have moved past. + if (this.#disposed || this.#state.conversationId !== conversationId) return + if (this.#turnsStarted !== turnsStarted) { + sendUserToast('That chat started another turn. Retry once it finishes.', true) + return + } + void this.sendRequest({ instructions: message.content }, replayInputs) + } + restartGeneration = () => {} + handleUserQuestionAnswer = () => false + handleToolConfirmation = () => {} + hasPendingRunForm = false + isRunFormPending = () => false + + // Copilot-only surfaces + mode = undefined + isSessionChat = false + supportsModelSettings = false + supportsMessageEditing = false + // The input's shape alone: whether this chat takes attachments at all is a fact about the + // flow, not about the workspace. Object storage decides whether it can right now, which is + // `attachmentsUnavailableReason` — a state on the control rather than a reason to move the + // input to the modal, where a file picker would be just as unable to upload. + get supportsMessageAttachments(): boolean { + return !!this.#options.attachmentsTarget?.() + } + get attachmentsUnavailableReason(): string | undefined { + return this.#options.attachmentsUnavailable?.() + } + // An AI agent step refuses a run with no `user_message`. + requiresMessageText = true + // Attachments go to object storage for the worker to read, so a linked folder — a live + // handle on the user's own disk — has no meaning here. + supportsLinkedFolders = false + attachmentsAsBlobs = true + // A single-file flow input takes one attachment per message. + get maxMessageAttachments(): number | undefined { + return this.#options.attachmentsTarget?.()?.multiple === false ? 1 : undefined + } + // What a provider actually takes. Anthropic's document block accepts base64 + // `application/pdf` and nothing else, so the wider set `is_document_mime` + // (windmill-ai/src/ai_types.rs) claims — csv, html, plain, docx, xlsx — is rejected with a + // 400 rather than read. Widen this only alongside a worker that inlines text as text. + attachmentAccept = 'image/*,application/pdf,.pdf' + tools = [] + // The enum's value, written out so this module never imports the copilot manager at + // runtime: its unit test would otherwise load the manager and the editor it pulls in. + autonomyMode = 'default' as AIAutonomyMode.DEFAULT + setAutonomyMode = () => {} + autoAcceptEditsActive = false + autoAcceptEditsAvailable = false + autoAcceptToolConfirmationsAvailable = false + planModeAvailable = false + attachedFiles = new AttachedFilesStore() + artifacts = new SessionArtifactsStore() +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts new file mode 100644 index 0000000000..53fd7b6ab9 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts @@ -0,0 +1,834 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { Chat, ChatMessage, ChatState } from 'windmill-chat' +import { FlowChatViewHost, toDisplayMessages } from './flowChatViewHost.svelte' + +vi.mock('$lib/gen', () => ({ + JobService: { getJobArgs: vi.fn() } +})) +vi.mock('$lib/toast', () => ({ sendUserToast: vi.fn() })) + +import { JobService } from '$lib/gen' +import { sendUserToast } from '$lib/toast' + +const getJobArgs = vi.mocked(JobService.getJobArgs) +const toast = vi.mocked(sendUserToast) + +beforeEach(() => { + getJobArgs.mockReset() + toast.mockReset() +}) + +function message(partial: Partial & Pick): ChatMessage { + return { + id: partial.id ?? `${partial.role}-${Math.random()}`, + content: '', + success: true, + createdAt: '2026-09-16T10:00:00Z', + pending: false, + ...partial + } +} + +function idleState(partial: Partial = {}): ChatState { + return { + conversationId: 'c1', + messages: [], + status: 'idle', + error: undefined, + conversations: [], + history: 'server', + loadingMessages: false, + hasMoreMessages: false, + ...partial + } +} + +/** A `Chat` whose state the test drives by hand. */ +function fakeChat(initial: ChatState = idleState()) { + let state = initial + const listeners = new Set<(s: ChatState) => void>() + const set = (patch: Partial) => { + state = { ...state, ...patch } + for (const listener of listeners) listener(state) + } + const chat = { + getState: () => state, + subscribe: (listener: (s: ChatState) => void) => { + listeners.add(listener) + listener(state) + return () => listeners.delete(listener) + }, + sendMessage: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + newConversation: vi.fn(), + selectConversation: vi.fn(async () => {}), + loadConversations: vi.fn(async () => []), + deleteConversation: vi.fn(async () => {}), + loadOlderMessages: vi.fn(async () => {}), + renameConversation: vi.fn(async () => {}), + destroy: vi.fn() + } satisfies Chat + return { chat, set } +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)) + +describe('toDisplayMessages', () => { + it('maps user, assistant and tool rows, marking the user message of a failed turn', () => { + const rows = [ + message({ role: 'user', content: 'hi' }), + message({ + role: 'assistant', + content: 'hello', + reasoning: 'thinking', + stepName: 'agent', + jobId: 'job-1', + createdAt: '2026-09-16T10:00:01Z' + }), + message({ role: 'user', content: 'again' }), + // A tool still running has no text yet: the server words the row on its return. + message({ + role: 'tool', + id: 'tool-row', + pending: true, + tool: { name: 'search', status: 'running', arguments: '{"q":"x"}' } + }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + const display = toDisplayMessages(rows) + expect(display[0]).toEqual({ + role: 'user', + index: 0, + content: 'hi', + error: undefined, + images: undefined, + contextElements: undefined + }) + expect(display[1]).toMatchObject({ + role: 'assistant', + content: 'hello', + reasoning: 'thinking', + jobId: 'job-1', + createdAt: '2026-09-16T10:00:01Z', + streaming: undefined + }) + expect(display[2]).toMatchObject({ role: 'user', index: 1, content: 'again', error: true }) + expect(display[3]).toMatchObject({ + role: 'tool', + tool_call_id: 'tool-row', + content: 'Running search', + toolName: 'search', + parameters: { q: 'x' }, + showDetails: true, + isLoading: true, + error: undefined + }) + }) + + it('shows a call the turn finished without as an error, not as still running', () => { + const display = toDisplayMessages([ + message({ role: 'user', content: 'hi' }), + message({ role: 'tool', tool: { name: 'search', status: 'running' } }) + ]) + expect(display[1]).toMatchObject({ + content: 'search did not finish', + error: 'search did not finish', + isLoading: false + }) + + const structured = toDisplayMessages([ + message({ role: 'user', content: 'hi' }), + message({ + role: 'tool', + tool: { name: 'structured_output', status: 'running', arguments: '{"n":1}' } + }) + ]) + expect(structured[1]).toMatchObject({ content: 'Running structured_output', error: undefined }) + }) + + it('shows the thinking that led to a call as its own card, and retries by transcript position', async () => { + const rows = [ + message({ role: 'user', content: 'first' }), + message({ + role: 'tool', + content: 'Used search tool', + reasoning: 'why', + tool: { name: 'search', status: 'success' } + }), + message({ role: 'assistant', content: 'done' }), + message({ role: 'user', content: 'second' }) + ] + const display = toDisplayMessages(rows) + expect(display.map((m) => [m.role, m.content])).toEqual([ + ['user', 'first'], + ['assistant', ''], + ['tool', 'Used search tool'], + ['assistant', 'done'], + ['user', 'second'] + ]) + expect(display[1]).toMatchObject({ reasoning: 'why' }) + expect(display[1]).not.toHaveProperty('streaming') + + const { chat } = fakeChat(idleState({ messages: rows })) + const host = new FlowChatViewHost(chat) + host.retryRequest(4) + await vi.waitFor(() => + expect(chat.sendMessage).toHaveBeenCalledWith('second', expect.anything()) + ) + host.dispose() + }) + + it('does not flag a turn whose tool failed but whose agent still answered', () => { + const display = toDisplayMessages([ + message({ role: 'user', content: 'try' }), + message({ + role: 'tool', + content: 'Error executing search', + success: false, + tool: { name: 'search', status: 'error' } + }), + message({ role: 'assistant', content: 'search is down, here is what I know' }) + ]) + expect(display[0]).toMatchObject({ role: 'user', error: undefined }) + }) + + it('offers no retry while the turn whose tool failed is still running', () => { + const messages = [ + message({ role: 'user', content: 'first' }), + message({ role: 'assistant', content: 'boom', success: false }), + message({ role: 'user', content: 'try' }), + message({ + role: 'tool', + content: 'Error executing search', + success: false, + tool: { name: 'search', status: 'error' } + }) + ] + const running = toDisplayMessages(messages, true) + expect(running[0]).toMatchObject({ role: 'user', error: true }) + expect(running[2]).toMatchObject({ role: 'user', error: undefined }) + expect(toDisplayMessages(messages, false)[2]).toMatchObject({ role: 'user', error: true }) + }) + + it('shows what a failed tool returned as its error, not the row label', () => { + const display = toDisplayMessages([ + message({ + role: 'tool', + content: 'Error executing lookup_stock', + success: false, + tool: { + name: 'lookup_stock', + status: 'error', + result: '{"message":"stock service unavailable","name":"Error","stack":"at main"}' + } + }), + message({ + role: 'tool', + content: 'Error executing mcp_search', + success: false, + tool: { name: 'mcp_search', status: 'error', result: 'connection refused' } + }) + ]) + expect(display[0]).toMatchObject({ + content: 'Error executing lookup_stock', + error: 'stock service unavailable' + }) + expect(display[1]).toMatchObject({ error: 'connection refused' }) + }) + + it('flags the streaming assistant message and a failed tool', () => { + const display = toDisplayMessages([ + message({ role: 'assistant', content: 'partial', pending: true }), + message({ + role: 'tool', + content: 'Error executing search', + success: false, + tool: { name: 'search', status: 'error' } + }) + ]) + expect(display[0]).toMatchObject({ role: 'assistant', streaming: true }) + expect(display[1]).toMatchObject({ + role: 'tool', + error: 'Error executing search', + showDetails: false, + isLoading: false + }) + }) + + // A row stored before the worker kept the call has only its sentence: the card names the + // tool and links its job, and has no details to open. + it('builds a tool card from the row alone, with its job link', () => { + const display = toDisplayMessages([ + message({ + role: 'tool', + content: 'Used lookup tool', + jobId: 'tool-job', + tool: { name: 'lookup', status: 'success', arguments: '{"id":1}', result: '"ok"' } + }), + message({ + role: 'tool', + content: 'Used search_docs tool', + jobId: 'older-job', + tool: { name: 'search_docs', status: 'success' } + }) + ]) + expect(display[0]).toMatchObject({ + toolName: 'lookup', + parameters: { id: 1 }, + result: 'ok', + showDetails: true, + jobId: 'tool-job' + }) + expect(display[1]).toMatchObject({ + toolName: 'search_docs', + parameters: undefined, + result: undefined, + showDetails: false, + jobId: 'older-job' + }) + }) + + it('labels answers with their step only once the transcript names more than one', () => { + const one = toDisplayMessages([ + message({ role: 'assistant', content: 'a', stepName: 'agent' }), + message({ role: 'assistant', content: 'b', stepName: 'agent' }) + ]) + expect(one.map((m) => (m.role === 'assistant' ? m.stepName : null))).toEqual([ + undefined, + undefined + ]) + const two = toDisplayMessages([ + message({ role: 'assistant', content: 'a', stepName: 'agent' }), + message({ role: 'assistant', content: 'b', stepName: 'reviewer' }) + ]) + expect(two.map((m) => (m.role === 'assistant' ? m.stepName : null))).toEqual([ + 'agent', + 'reviewer' + ]) + }) + + it('shows the files a user message carried, read from the message itself', () => { + const display = toDisplayMessages( + [ + message({ + role: 'user', + content: 'go', + attachments: [ + { input: 'user_attachments', s3: 'chat/u1/shot.png' }, + { input: 'user_attachments', s3: 'chat/u1/notes.pdf' } + ] + }) + ], + false, + new Set(), + { workspace: 'ws' } + ) + expect(display[0]).toMatchObject({ + role: 'user', + images: [{ name: 'shot.png' }], + contextElements: [{ title: 'notes.pdf' }] + }) + }) +}) + +describe('FlowChatViewHost', () => { + it('sends the text with the additional inputs and reports loading from the status', async () => { + const { chat, set } = fakeChat() + const host = new FlowChatViewHost(chat, { additionalInputs: () => ({ tone: 'brief' }) }) + expect(host.loading).toBe(false) + expect(await host.sendRequest({ instructions: ' hello ' })).toBe(true) + expect(chat.sendMessage).toHaveBeenCalledWith('hello', { + inputs: { tone: 'brief' }, + attachments: [], + attachmentsInput: undefined + }) + expect(await host.sendRequest({ instructions: ' ' })).toBe(false) + set({ status: 'streaming' }) + expect(host.loading).toBe(true) + set({ status: 'idle' }) + expect(host.loading).toBe(false) + host.dispose() + }) + + it('queues a message typed during a turn and sends it once the turn is released', async () => { + const { chat, set } = fakeChat() + // The chat publishes `idle` from inside `sendMessage`, before that call returns and + // releases the turn; a flush in between is refused as a second turn. + let releaseTurn = () => {} + chat.sendMessage.mockImplementationOnce( + () => new Promise((resolve) => (releaseTurn = resolve)) + ) + const host = new FlowChatViewHost(chat) + void host.sendRequest({ instructions: 'start' }) + set({ status: 'streaming' }) + host.queueMessage('first') + host.queueMessage('second') + expect(host.queuedMessage).toBe('first\nsecond') + set({ status: 'idle' }) + await flush() + expect(chat.sendMessage).toHaveBeenCalledTimes(1) + releaseTurn() + await flush() + expect(host.queuedMessage).toBe('') + expect(chat.sendMessage).toHaveBeenLastCalledWith('first\nsecond', { + inputs: undefined, + attachments: [], + attachmentsInput: undefined + }) + host.dispose() + }) + + it('hands the queue back when a failed flow settles as idle', () => { + const { chat, set } = fakeChat( + idleState({ status: 'streaming', messages: [message({ role: 'user', content: 'go' })] }) + ) + const host = new FlowChatViewHost(chat) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + host.queueMessage('later') + // A failed flow is answered with its error and the status still returns to idle. + set({ + status: 'idle', + messages: [ + message({ role: 'user', content: 'go' }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + }) + expect(prependText).toHaveBeenCalledWith('later', [], [], []) + expect(chat.sendMessage).not.toHaveBeenCalled() + host.dispose() + }) + + it('hands the queue back instead of sending while sending is disabled', async () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + let deploying = false + const host = new FlowChatViewHost(chat, { sendDisabled: () => deploying }) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + host.queueMessage('after deploy') + // A deployment starts while the turn is still running; the composer is disabled. + deploying = true + set({ status: 'idle' }) + await flush() + expect(chat.sendMessage).not.toHaveBeenCalled() + expect(prependText).toHaveBeenCalledWith('after deploy', [], [], []) + expect(host.queuedMessage).toBe('') + host.dispose() + }) + + it('drops a flush still waiting on the turn once disposed', async () => { + const { chat, set } = fakeChat() + let releaseTurn = () => {} + chat.sendMessage.mockImplementationOnce( + () => new Promise((resolve) => (releaseTurn = resolve)) + ) + const host = new FlowChatViewHost(chat) + void host.sendRequest({ instructions: 'start' }) + set({ status: 'streaming' }) + host.queueMessage('never') + set({ status: 'idle' }) + host.dispose() + releaseTurn() + await flush() + expect(chat.sendMessage).toHaveBeenCalledTimes(1) + }) + + it('hands the text back when the chat refuses the turn', async () => { + const { chat } = fakeChat() + chat.sendMessage.mockRejectedValueOnce(new Error('a message is already being answered')) + const host = new FlowChatViewHost(chat) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + await host.sendRequest({ instructions: 'kept' }) + expect(prependText).toHaveBeenCalledWith('kept', [], [], []) + host.dispose() + }) + + it('hands the queue back to the composer on Stop and on a failed turn', async () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + const host = new FlowChatViewHost(chat) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + host.queueMessage('later') + host.cancel() + expect(chat.stop).toHaveBeenCalled() + expect(prependText).toHaveBeenCalledWith('later', [], [], []) + expect(host.queuedMessage).toBe('') + + host.queueMessage('after error') + set({ status: 'error' }) + expect(prependText).toHaveBeenLastCalledWith('after error', [], [], []) + expect(chat.sendMessage).not.toHaveBeenCalled() + host.dispose() + }) + + const PNG = `data:image/png;base64,${btoa('\x89PNG')}` + const image = { name: 'shot.webp', dataUrl: PNG, mediaType: 'image/png' } as any + const pdf = { + name: 'contract.pdf', + dataUrl: `data:application/pdf;base64,${btoa('%PDF')}`, + mediaType: 'application/pdf', + size: 4 + } + const listInput = { name: 'files', multiple: true } + + it('takes attachments only where the flow has an input for them', () => { + const { chat } = fakeChat() + const none = new FlowChatViewHost(chat) + expect(none.supportsMessageAttachments).toBe(false) + const list = new FlowChatViewHost(chat, { attachmentsTarget: () => listInput }) + expect(list.supportsMessageAttachments).toBe(true) + expect(list.maxMessageAttachments).toBeUndefined() + const single = new FlowChatViewHost(chat, { + attachmentsTarget: () => ({ name: 'file', multiple: false }), + attachmentsUnavailable: () => 'no storage' + }) + expect(single.maxMessageAttachments).toBe(1) + expect(single.attachmentsUnavailableReason).toBe('no storage') + }) + + it('hands the attachments to the chat, and drops a stored value for their input', async () => { + const { chat } = fakeChat() + const host = new FlowChatViewHost(chat, { + additionalInputs: () => ({ tone: 'brief', files: [{ s3: 'stale' }] }), + attachmentsTarget: () => listInput + }) + await host.sendRequest({ instructions: 'read', images: [image], blobs: [pdf] }) + const [, options] = chat.sendMessage.mock.calls[0] as any + expect(options.inputs).toEqual({ tone: 'brief' }) + expect(options.attachmentsInput).toBe(listInput) + expect(options.attachments).toEqual([ + { name: 'shot.webp', data: PNG, mediaType: 'image/png' }, + { name: 'contract.pdf', data: pdf.dataUrl, mediaType: 'application/pdf' } + ]) + }) + + // A queue merged over several turns reaches the host as one send. + it('re-applies a single-file cap to a merged queue', async () => { + const { chat } = fakeChat() + const host = new FlowChatViewHost(chat, { + attachmentsTarget: () => ({ name: 'file', multiple: false }) + }) + await host.sendRequest({ instructions: 'read', images: [image], blobs: [pdf] }) + const [, options] = chat.sendMessage.mock.calls[0] as any + expect(options.attachments.map((a: any) => a.name)).toEqual(['shot.webp']) + }) + + it('hands the draft back with its attachments when the upload is refused or stopped', async () => { + const { chat } = fakeChat() + const host = new FlowChatViewHost(chat, { attachmentsTarget: () => listInput }) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + chat.sendMessage.mockRejectedValueOnce(new Error('POST upload failed (500)')) + await host.sendRequest({ instructions: 'read', blobs: [pdf] }) + expect(prependText).toHaveBeenCalledWith('read', [], [], [pdf]) + chat.sendMessage.mockRejectedValueOnce(new DOMException('aborted', 'AbortError')) + await host.sendRequest({ instructions: 'again', images: [image] }) + expect(prependText).toHaveBeenLastCalledWith('again', [image], [], []) + host.dispose() + }) + + // The chat settles a withdrawn turn as `idle`, which reads like a turn that ran; a queue + // left waiting would then go out with the failed draft merged in front of it. + it('does not send what was queued behind an upload that failed', async () => { + const { chat, set } = fakeChat( + idleState({ messages: [message({ role: 'user', content: 'earlier' })] }) + ) + let refuse = (_e: Error) => {} + chat.sendMessage.mockImplementationOnce( + () => new Promise((_, reject) => (refuse = reject)) + ) + const host = new FlowChatViewHost(chat, { attachmentsTarget: () => listInput }) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + void host.sendRequest({ instructions: 'A', blobs: [pdf] }) + set({ status: 'submitted' }) + host.queueMessage('B') + set({ status: 'idle' }) + refuse(new Error('upload failed (500)')) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(chat.sendMessage).toHaveBeenCalledTimes(1) + expect(host.queuedMessage).toBe('') + expect(prependText.mock.calls.map((c) => c[0])).toEqual(['B', 'A']) + expect(prependText).toHaveBeenLastCalledWith('A', [], [], [pdf]) + host.dispose() + }) + + it('retries a failed turn with the files its run had, uploading nothing', async () => { + const { chat } = fakeChat( + idleState({ + messages: [ + message({ role: 'user', content: 'read', jobId: 'flow-job' }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + }) + ) + getJobArgs.mockResolvedValueOnce({ + user_message: 'read', + user_attachments: [{ s3: 'chat/u1/contract.pdf', filename: 'contract.pdf' }] + } as any) + const host = new FlowChatViewHost(chat, { + workspace: () => 'ws', + attachmentsTarget: () => listInput + }) + await host.retryRequest(0) + const [text, options] = chat.sendMessage.mock.calls[0] as any + expect(text).toBe('read') + // The references the run already has, so the files are not uploaded a second time. + expect(options.inputs).toEqual({ + user_attachments: [{ s3: 'chat/u1/contract.pdf', filename: 'contract.pdf' }] + }) + expect(options.attachments).toEqual([]) + host.dispose() + }) + + it('refuses a message without a file when the flow requires one', async () => { + const { chat } = fakeChat() + const host = new FlowChatViewHost(chat, { + attachmentsTarget: () => ({ ...listInput, required: true }) + }) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + expect(await host.sendRequest({ instructions: 'no file' })).toBe(false) + expect(chat.sendMessage).not.toHaveBeenCalled() + expect(prependText).toHaveBeenCalledWith('no file', [], [], []) + expect(await host.sendRequest({ instructions: 'with file', blobs: [pdf] })).toBe(true) + expect(chat.sendMessage).toHaveBeenCalledTimes(1) + host.dispose() + }) + + it('queues attachments with the text and sends them together', async () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + const host = new FlowChatViewHost(chat, { attachmentsTarget: () => listInput }) + host.queueMessage('look', [image], undefined, undefined, [pdf]) + expect(host.queuedImages).toEqual([image]) + expect(host.queuedBlobs).toEqual([pdf]) + set({ status: 'idle' }) + await new Promise((resolve) => setTimeout(resolve, 0)) + const [text, options] = chat.sendMessage.mock.calls[0] as any + expect(text).toBe('look') + expect(options.attachments.map((a: any) => a.name)).toEqual(['shot.webp', 'contract.pdf']) + expect(host.queuedBlobs).toEqual([]) + host.dispose() + }) + + it('offers no retry on a turn the reader stopped', () => { + const failedTool = message({ + role: 'tool', + content: 'Error executing search', + success: false, + tool: { name: 'search', status: 'error' } + }) + const { chat, set } = fakeChat( + idleState({ + status: 'streaming', + messages: [message({ id: 'live', role: 'user', content: 'go' }), failedTool] + }) + ) + const host = new FlowChatViewHost(chat) + host.cancel() + expect(chat.stop).toHaveBeenCalled() + set({ status: 'idle' }) + expect(host.displayMessages[0]).toMatchObject({ role: 'user', error: undefined }) + // The chat re-reads the rows: the user message gets its row id, and the cancelled + // flow's failure lands as the answer. + const cancelled = message({ role: 'assistant', content: 'Job canceled', success: false }) + set({ + messages: [ + message({ id: 'live', serverId: 'row-2', role: 'user', content: 'go' }), + failedTool, + cancelled + ] + }) + expect(host.displayMessages[0]).toMatchObject({ role: 'user', error: undefined }) + // Reopened, with an older page that failed in front: messages are rebuilt from rows. + set({ + messages: [ + message({ id: 'row-0', serverId: 'row-0', role: 'user', content: 'before' }), + message({ role: 'assistant', content: 'boom', success: false }), + message({ id: 'row-2', serverId: 'row-2', role: 'user', content: 'go' }), + failedTool, + cancelled + ] + }) + expect(host.displayMessages[0]).toMatchObject({ content: 'before', error: true }) + expect(host.displayMessages[2]).toMatchObject({ content: 'go', error: undefined }) + // The next turn is not stopped and fails on its own. + set({ + messages: [ + ...chat.getState().messages, + message({ role: 'user', content: 'again' }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + }) + expect(host.displayMessages.at(-2)).toMatchObject({ role: 'user', error: true }) + host.dispose() + }) + + it('stops following the chat once disposed', () => { + const { chat, set } = fakeChat() + const host = new FlowChatViewHost(chat) + host.dispose() + set({ status: 'streaming' }) + expect(host.loading).toBe(false) + }) + + describe('retry', () => { + const failedTurn = () => + idleState({ + messages: [ + message({ role: 'user', content: 'go', jobId: 'flow-job' }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + }) + + it("replays the turn with the inputs its run had, not the composer's", async () => { + const { chat } = fakeChat(failedTurn()) + getJobArgs.mockResolvedValueOnce({ user_message: 'go', tone: 'terse', model: 'old' } as any) + const host = new FlowChatViewHost(chat, { + workspace: () => 'ws', + additionalInputs: () => ({ tone: 'brief', model: 'new' }), + inputsShownInComposer: () => ['model'] + }) + await host.retryRequest(0) + expect(getJobArgs).toHaveBeenCalledWith({ workspace: 'ws', id: 'flow-job' }) + // The composer's own control wins for what it shows; everything else replays. + expect(chat.sendMessage).toHaveBeenCalledWith( + 'go', + expect.objectContaining({ inputs: { tone: 'terse', model: 'new' } }) + ) + host.dispose() + }) + + it('falls back to a plain resend once the job is purged', async () => { + const { chat } = fakeChat(failedTurn()) + getJobArgs.mockRejectedValueOnce(Object.assign(new Error('gone'), { status: 404 })) + const host = new FlowChatViewHost(chat, { + workspace: () => 'ws', + additionalInputs: () => ({ tone: 'brief' }) + }) + await host.retryRequest(0) + expect(chat.sendMessage).toHaveBeenCalledWith( + 'go', + expect.objectContaining({ inputs: { tone: 'brief' } }) + ) + expect(toast).not.toHaveBeenCalled() + host.dispose() + }) + + it('does nothing but say so when the run cannot be read', async () => { + const { chat } = fakeChat(failedTurn()) + getJobArgs.mockRejectedValueOnce(Object.assign(new Error('down'), { status: 500 })) + const host = new FlowChatViewHost(chat, { workspace: () => 'ws' }) + await host.retryRequest(0) + expect(chat.sendMessage).not.toHaveBeenCalled() + expect(toast).toHaveBeenCalledWith('Could not read what that turn ran with. Try again.', true) + host.dispose() + }) + + it('refuses once a turn started while the run was being read', async () => { + const { chat, set } = fakeChat(failedTurn()) + let answer = (_: unknown) => {} + getJobArgs.mockImplementationOnce(() => new Promise((resolve) => (answer = resolve)) as any) + const host = new FlowChatViewHost(chat, { workspace: () => 'ws' }) + const retried = host.retryRequest(0) + set({ status: 'streaming' }) + answer({ user_message: 'go' }) + await retried + expect(chat.sendMessage).not.toHaveBeenCalled() + expect(toast).toHaveBeenCalledWith( + 'That chat started another turn. Retry once it finishes.', + true + ) + host.dispose() + }) + + // A quick turn can start and settle inside that read: the chat is idle again, but its + // latest message is not the one this retry was clicked on. + it('refuses once a turn ran and settled while the run was being read', async () => { + const { chat, set } = fakeChat(failedTurn()) + let answer = (_: unknown) => {} + getJobArgs.mockImplementationOnce(() => new Promise((resolve) => (answer = resolve)) as any) + const host = new FlowChatViewHost(chat, { workspace: () => 'ws' }) + const retried = host.retryRequest(0) + set({ status: 'streaming' }) + set({ status: 'idle' }) + answer({ user_message: 'go' }) + await retried + expect(chat.sendMessage).not.toHaveBeenCalled() + expect(toast).toHaveBeenCalledWith( + 'That chat started another turn. Retry once it finishes.', + true + ) + host.dispose() + }) + }) + + describe('streaming reveal', () => { + /** A scheduler the test steps by hand, and a clock it advances. */ + function manualReveal() { + let time = 0 + const queue: (() => void)[] = [] + return { + options: { + instant: false, + now: () => time, + schedule: (cb: () => void) => (queue.push(cb), cb), + cancel: (handle: unknown) => { + const i = queue.indexOf(handle as () => void) + if (i >= 0) queue.splice(i, 1) + } + }, + tick: (ms: number) => { + time += ms + const due = queue.splice(0) + for (const cb of due) cb() + } + } + } + + it('paces a streaming answer and shows it whole once it settles', () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + const reveal = manualReveal() + const host = new FlowChatViewHost(chat, { revealOptions: reveal.options }) + const streaming = message({ role: 'assistant', id: 'a1', content: '', pending: true }) + set({ messages: [{ ...streaming, content: 'The answer, in one burst of text.' }] }) + const shown = () => (host.displayMessages[0] as { content: string }).content + // Nothing is revealed until the pacer's first frame, and that frame shows a slice. + expect(shown()).toBe('') + reveal.tick(16) + expect(shown().length).toBeGreaterThan(0) + expect(shown().length).toBeLessThan('The answer, in one burst of text.'.length) + expect('The answer, in one burst of text.'.startsWith(shown())).toBe(true) + // Settled: the whole text, whatever the pacer had got to. + set({ + status: 'idle', + messages: [{ ...streaming, content: 'The answer, in one burst of text.', pending: false }] + }) + expect(shown()).toBe('The answer, in one burst of text.') + host.dispose() + }) + + it('shows a paced row whole once a tool card follows it', () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + const reveal = manualReveal() + const host = new FlowChatViewHost(chat, { revealOptions: reveal.options }) + const answer = message({ + role: 'assistant', + id: 'a1', + content: 'Let me look.', + pending: true + }) + set({ messages: [answer] }) + expect((host.displayMessages[0] as { content: string }).content).toBe('') + set({ + messages: [ + answer, + message({ role: 'tool', pending: true, tool: { name: 'lookup', status: 'running' } }) + ] + }) + expect((host.displayMessages[0] as { content: string }).content).toBe('Let me look.') + host.dispose() + }) + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/messageAttachments.test.ts b/frontend/src/lib/components/flows/conversations/messageAttachments.test.ts new file mode 100644 index 0000000000..2ea1d5e28b --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/messageAttachments.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' +import { attachmentLanes } from './messageAttachments' + +describe('attachmentLanes', () => { + it('splits images into thumbnails and everything else into named file chips', () => { + const { images, contextElements } = attachmentLanes('ws', [ + { input: 'user_attachments', s3: 'chat/u1/shot.PNG', storage: 'secondary' }, + { input: 'user_attachments', s3: 'chat/u1/0/notes.pdf', filename: 'Q3 notes.pdf' }, + { input: 'report', s3: 'chat/u1/data.csv' } + ]) + expect(images.map((i) => i.name)).toEqual(['shot.PNG']) + expect(images[0].dataUrl).toContain('/api/w/ws/job_helpers/download_s3_file?') + expect(images[0].dataUrl).toContain('file_key=chat%2Fu1%2Fshot.PNG') + expect(images[0].dataUrl).toContain('storage=secondary') + expect(contextElements.map((c) => c.title)).toEqual(['Q3 notes.pdf', 'data.csv']) + }) + + it('shows an image as a file chip when there is no workspace to link it in', () => { + const { images, contextElements } = attachmentLanes(undefined, [ + { input: 'user_attachments', s3: 'chat/u1/shot.png' } + ]) + expect(images).toEqual([]) + expect(contextElements.map((c) => c.title)).toEqual(['shot.png']) + }) +}) diff --git a/frontend/src/lib/components/flows/conversations/messageAttachments.ts b/frontend/src/lib/components/flows/conversations/messageAttachments.ts new file mode 100644 index 0000000000..36c7d103c3 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/messageAttachments.ts @@ -0,0 +1,48 @@ +/** The files a user message carried, as the lanes its bubble renders: image thumbnails and file chips. */ +import type { ChatAttachment } from 'windmill-chat' +import { base } from '$lib/base' +import { + createAttachedFileContextElement, + type ContextElement +} from '$lib/components/copilot/chat/context' +import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils' + +const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg', '.avif'] + +export type MessageAttachments = { images: AttachedImage[]; contextElements: ContextElement[] } + +function displayName(attachment: ChatAttachment): string { + return attachment.filename || attachment.s3.split('/').pop() || attachment.s3 +} + +function looksLikeImage(attachment: ChatAttachment): boolean { + const name = displayName(attachment).toLowerCase() + return IMAGE_EXTENSIONS.some((ext) => name.endsWith(ext)) +} + +/** Same-origin, cookie-authed GET, usable directly as an . */ +function downloadUrl(workspace: string, attachment: ChatAttachment): string { + const params = new URLSearchParams({ file_key: attachment.s3 }) + if (attachment.storage) params.set('storage', attachment.storage) + return `${base}/api/w/${workspace}/job_helpers/download_s3_file?${params.toString()}` +} + +/** Without a workspace there is no link to build, so an image falls back to a file chip. */ +export function attachmentLanes( + workspace: string | undefined, + attachments: readonly ChatAttachment[] | undefined +): MessageAttachments { + const images: AttachedImage[] = [] + const contextElements: ContextElement[] = [] + for (const attachment of attachments ?? []) { + const name = displayName(attachment) + if (workspace && looksLikeImage(attachment)) { + images.push({ dataUrl: downloadUrl(workspace, attachment), mediaType: 'image/png', name }) + } else { + contextElements.push( + createAttachedFileContextElement(name, `Attached file · ${attachment.s3}`) + ) + } + } + return { images, contextElements } +} diff --git a/frontend/src/lib/components/flows/flowDeleteController.ts b/frontend/src/lib/components/flows/flowDeleteController.ts index a1933ba080..d7d98fadda 100644 --- a/frontend/src/lib/components/flows/flowDeleteController.ts +++ b/frontend/src/lib/components/flows/flowDeleteController.ts @@ -1,4 +1,5 @@ import type { GroupDisplayState } from '$lib/components/graph/groupEditor.svelte' +import { refreshStateStore } from '$lib/svelte5Utils.svelte' import type { GroupedModulesProxy } from '$lib/components/graph/groupedModulesProxy.svelte' import type { SelectionManager } from '$lib/components/graph/selectionUtils.svelte' import type { FlowStructureNode } from '$lib/components/graph/flowStructure' @@ -8,7 +9,6 @@ import type { StateStore } from '$lib/utils' import { createDeletePlan, removeDeletePlanTools, type DeletePlan } from './flowDeleteUtils' import type { FlowState } from './flowState' import { deleteFlowStateById } from './flowStateUtils.svelte' -import { refreshFlowStateStore } from './flowStoreRefresh.svelte' export type PreparedDeleteRequest = { plan: DeletePlan @@ -74,7 +74,7 @@ export function executeDeletePlan( deleteFlowStateById(id, args.flowStateStore) } - refreshFlowStateStore(args.flowStore) + refreshStateStore(args.flowStore) if (plan.inputIds.length === 1) { args.onDelete?.(plan.targets[0].id) diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index 4d6cad4df5..13f1ae3ef5 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -4,6 +4,25 @@ import type { Schema } from '$lib/common' import { emptySchema } from '$lib/utils' import type { FlowModule, InputTransform } from '$lib/gen' import { AGENT_FLOW_LOCAL_KEYS } from './agentResourceUtils' +import { AGENT_HISTORY_KEYS } from './agentFormFields' + +/** Display names for the memory field's options, so anything else naming the setting an author + * picked cannot drift from the button they see. */ +export const MEMORY_OPTION_LABELS: Record = { + off: 'Off', + window: 'On', + auto: 'On (legacy)', + manual: 'Previous messages (legacy)' +} + +export function memoryOptionLabel(memory: any): string | undefined { + // Managed memory that keeps no messages runs as off, and a note about what that state reads + // must say so. + if ((memory?.kind === 'window' || memory?.kind === 'auto') && !memory.context_length) { + return MEMORY_OPTION_LABELS.off + } + return memory?.kind ? MEMORY_OPTION_LABELS[memory.kind] : undefined +} export const AI_AGENT_SCHEMA: Schema = { $schema: 'https://json-schema.org/draft/2020-12/schema', @@ -15,159 +34,157 @@ export const AI_AGENT_SCHEMA: Schema = { output_type: { type: 'string', description: - 'The type of output the AI agent will generate (text or image). Image output requires a configured workspace S3 storage, will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.', + 'Whether the answer is text or an image. An image needs S3 storage on the workspace, and ignores tools.', enum: ['text', 'image'], default: 'text' }, user_message: { type: 'string', - description: - 'The message to give as input to the AI agent. Optional when messages array is provided. You can turn on chat input mode on the input interface to link this field to the message sent by the user.' + description: 'The message sent to the agent as the user turn.' }, system_prompt: { type: 'string', - description: 'The system prompt to give as input to the AI agent.' + description: 'Sets how the agent behaves. Sent ahead of everything else.', + // The one field people write paragraphs into, so it opens as a text area. + minRows: 5, + placeholder: + "You are a support agent.\nLook up an answer with your tools before replying.\nCite what you used, and say you don't know rather than guessing." }, streaming: { type: 'boolean', - description: 'Whether to stream the output of the AI agent.', + description: 'Stream the answer as it is produced.', default: true, - showExpr: "fields.output_type === 'text'" + showExpr: "fields.output_type !== 'image'" }, memory: { type: 'object', description: - 'Configure how conversation memory is managed. Choose "auto" to let Windmill automatically store and load messages (up to N last messages), or "manual" to provide an explicit array of conversation messages. The system_prompt and user_message are added to the messages if provided.', + 'Windmill stores the conversation and sends its last messages with each request.', + enumLabels: MEMORY_OPTION_LABELS, + // Chat mode keys memory on the conversation, so a chat whose agent has memory off + // forgets every turn. Enabling chat mode turns it on; this keeps it there. A step + // sitting at `off` stays switchable, or a flow that reached that state before — + // an agent added to an already-chat-enabled flow — would have no way out of it. + lockOneOfWhenChatEnabled: + "Chat mode keys this agent's history on the conversation, so memory stays on while it is enabled.", oneOf: [ { type: 'object', title: 'off', properties: { - kind: { - type: 'string', - enum: ['off'], - description: 'Disable conversation memory' - } + kind: { type: 'string', enum: ['off'] } } }, { type: 'object', - title: 'auto', + title: 'window', properties: { - kind: { - type: 'string', - enum: ['auto'], - default: 'auto', - description: 'Automatically manage conversation history' - }, + kind: { type: 'string', enum: ['window'] }, context_length: { type: 'number', - description: - 'Number of most recent messages to store and load. Set to 0 to disable memory.', - default: 5 - }, - memory_id: { - type: 'string', - format: 'uuid', - 'x-auto-generate': true, - description: - 'Custom memory identifier. Each unique ID maintains separate conversation history.', - hideWhenChatEnabled: true + title: 'Messages to keep', + description: 'Number of most recent messages to load and store. 0 turns memory off.', + default: 10 } }, - required: ['kind'], - 'x-no-s3-storage-workspace-warning': - 'When no S3 storage is configured in your workspace settings, memory will be stored in database, which implies a limit of 100KB per memory entry. If you need to store more messages, you should use S3 storage in your workspace settings.' - }, - { - type: 'object', - title: 'manual', - properties: { - kind: { - type: 'string', - enum: ['manual'], - description: - 'Manually provide conversation messages, bypassing automatic memory management' - }, - messages: { - type: 'array', - description: 'Array of conversation messages to use as history', - items: { - type: 'object', - properties: { - role: { - type: 'string', - enum: ['user', 'assistant', 'system'] - }, - content: { - type: 'string' - }, - tool_calls: { - type: 'array', - nullable: true, - items: { - type: 'object', - properties: { - id: { type: 'string' }, - type: { type: 'string' }, - function: { - type: 'object', - properties: { - name: { type: 'string' }, - arguments: { type: 'string' } - } - } - } - } - }, - tool_call_id: { - type: 'string', - nullable: true, - description: 'The ID of the tool call this message is responding to' + required: ['kind', 'context_length'] + } + ], + showExpr: "fields.output_type !== 'image'" + }, + memory_id: { + type: 'string', + description: + 'Names the memory this step reads and writes, overriding the memory id the run was started with. Read only while managed memory is on, and not at all by an older auto or manual memory.', + showExpr: "fields.output_type !== 'image'" + }, + previous_messages: { + type: 'array', + description: + 'History the flow supplies, sent before the user message. Read only while managed memory is off, and not at all by an older auto or manual memory.', + items: { + type: 'object', + properties: { + role: { + type: 'string', + enum: ['user', 'assistant', 'system'] + }, + content: { + type: 'string' + }, + tool_calls: { + type: 'array', + nullable: true, + items: { + type: 'object', + properties: { + id: { type: 'string' }, + type: { type: 'string' }, + function: { + type: 'object', + properties: { + name: { type: 'string' }, + arguments: { type: 'string' } } - }, - required: ['role'] + } } } }, - required: ['kind', 'messages'] - } - ], - showExpr: "fields.output_type === 'text'" + tool_call_id: { + type: 'string', + nullable: true, + description: 'The ID of the tool call this message is responding to' + } + }, + required: ['role'] + }, + showExpr: "fields.output_type !== 'image'" }, output_schema: { type: 'object', - description: 'JSON schema that the AI agent will follow for its response format.', + description: 'A JSON schema the answer has to follow.', format: 'json-schema', - showExpr: "fields.output_type === 'text'" + showExpr: "fields.output_type !== 'image'" }, user_attachments: { type: 'array', - description: - 'Array of files (images or PDFs) to give as input to the AI agent. Requires a configured workspace S3 storage.', + description: 'Images or PDFs sent with the message. Needs S3 storage on the workspace.', items: { type: 'object', resourceType: 's3object' } }, + // The step's own roster fills `items.enum` in, so the static editor offers the tools this + // agent actually has (`AiAgentStepInputs`). Absence, not an empty list, is what carries every + // tool: a step that holds the field and names nothing has chosen to advertise none. + // Shown for image output as the roster it narrows is, even though neither is used there. + enabled_tools: { + type: 'array', + // Deliberately short. It is the only place the field's text is always on screen rather than + // behind the row's tooltip, and the surface it shows on is the run form, which offers the + // names in a picker and has no unset state to explain. + description: 'Which of the agent tools a run may call.', + items: { + type: 'string' + } + }, max_completion_tokens: { type: 'number', - description: 'The maximum number of output tokens.' + description: 'The most tokens the answer may use.' }, temperature: { type: 'number', - description: - 'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).', - showExpr: "fields.output_type === 'text'" + description: 'How random the generation is, from 0 for deterministic up to 2.' }, max_iterations: { type: 'number', - description: - 'Limits how many times the agent can loop through reasoning and tool use. Range: 1-1000.', + description: 'How many times the agent may loop over calling the model and running tools.', default: 10 } }, - required: ['provider', 'output_type'], + // `output_type` defaults to text on the backend, so leaving it unset is valid: the form drops + // the row rather than showing a field whose value a run would ignore. + required: ['provider'], type: 'object', order: [ 'provider', @@ -176,14 +193,59 @@ export const AI_AGENT_SCHEMA: Schema = { 'system_prompt', 'streaming', 'memory', + 'memory_id', + 'previous_messages', 'output_schema', 'user_attachments', + 'enabled_tools', 'max_completion_tokens', 'temperature', 'max_iterations' ] } +/** Memory shapes older editors wrote. The step form offers one only to a step that still holds it, + * since the one-of field rewrites a value that matches none of its options. No field carries a + * default: the form writes one into a missing field on open, and a missing count runs as off. */ +export const LEGACY_MEMORY_VARIANTS: Record = { + auto: { + type: 'object', + title: 'auto', + properties: { + kind: { type: 'string', enum: ['auto'] }, + context_length: { type: 'number', title: 'Messages to keep' }, + memory_id: { type: 'string', title: 'Fixed memory id' } + }, + required: ['kind'] + }, + manual: { + type: 'object', + title: 'manual', + properties: { + kind: { type: 'string', enum: ['manual'] }, + messages: { type: 'array', items: AI_AGENT_SCHEMA.properties?.previous_messages?.items } + }, + required: ['kind', 'messages'] + } +} + +/** The memory property to render for a value: a legacy kind is added as an option only while the + * value holds it. Otherwise the property itself is returned, which callers compare by identity to + * avoid rebuilding the step schema. */ +export function memoryPropertyFor(property: any, value: any, chatInputEnabled = false): any { + let legacy = value?.kind ? LEGACY_MEMORY_VARIANTS[value.kind] : undefined + if (!legacy || !property?.oneOf) return property + // The baked id field is offered only to a value saved with the key (by presence, not content, + // or clearing it to retype would remove the field mid-edit), and never in a chat flow, which + // runs on the conversation id and drops it on save; there the nested form removes the key on + // open, as the hidden field did before. + if (value.kind === 'auto' && (chatInputEnabled || !('memory_id' in value))) { + const { memory_id: _, ...properties } = legacy.properties + legacy = { ...legacy, properties } + } + return { ...property, oneOf: [...property.oneOf, legacy] } +} + function migrateAiAgentInputTransforms( inputTransforms: Record ): Record { @@ -285,13 +347,19 @@ export async function loadSchemaFromModule( : Object.keys(AI_AGENT_SCHEMA.properties ?? {}) return { input_transforms: keys.reduce((accu, key) => { - accu[key] = input_transforms[key] ?? { - type: 'static', - value: undefined - } + const transform = + input_transforms[key] ?? + // A present history input is the step's choice at runtime, so it gets no placeholder. + ((AGENT_HISTORY_KEYS as readonly string[]).includes(key) + ? undefined + : { type: 'static', value: undefined }) + if (transform) accu[key] = transform return accu }, {}), - schema: AI_AGENT_SCHEMA + // A copy per step, never the shared constant: the form writes back into the property it + // renders (`InputTransformForm` binds `schema.properties[argName]`), and the tool names + // one step offers would otherwise become every step's. + schema: structuredClone(AI_AGENT_SCHEMA) } } diff --git a/frontend/src/lib/components/flows/flowState.ts b/frontend/src/lib/components/flows/flowState.ts index d70ee38f55..595227f57d 100644 --- a/frontend/src/lib/components/flows/flowState.ts +++ b/frontend/src/lib/components/flows/flowState.ts @@ -5,6 +5,7 @@ import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils' import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte' +import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts' import { loadFlowModuleState } from './flowStateUtils.svelte' import { emptyFlowModuleState } from './utils.svelte' import type { StateStore } from '$lib/utils' @@ -90,10 +91,15 @@ async function mapFlowModule( // the graph can render its tool nodes. They are display-only (their inputs are edited in // the step panel, which infers schemas itself), so no per-tool module state is loaded — // resource tool ids are not flow-unique and must not key into the flow state. - await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id) + // Drafts included: every caller of `initFlowState` is a flow editor, where the graph has + // to show the tools a test would run. Read-only viewers publish for themselves. + await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true) } else { + // Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's + // own state below, leaving it with no schema rather than with no tool schemas. + const tools = Array.isArray(value.tools) ? value.tools : [] await Promise.all( - (value.tools ?? []).filter(isFlowModuleTool).map(async (tool) => { + tools.filter(isFlowModuleTool).map(async (tool) => { modulesState[tool.id] = await loadFlowModuleState(agentToolToFlowModule(tool), workspace) }) ) @@ -116,13 +122,19 @@ export async function publishLinkedAgentTools( agentRef: string, workspace: string | undefined, scope: string, - moduleId: string + moduleId: string, + /** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph + * shows the tool set a test would run; read-only viewers pass false, since a run they are + * displaying used the deployed agent. Required rather than defaulted: an editor call site that + * forgets it republishes the deployed tools over the drafted ones, which reads as the graph + * spontaneously reverting. */ + withDraft: boolean ) { const genKey = `${scope}:${moduleId}` const gen = claimLinkedToolsFetch(scope, moduleId) - const tools = await resolveLinkedAgentTools(agentRef, workspace) + const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft) if (linkedToolFetchGen.get(genKey) === gen) { - setLinkedAgentTools(scope, moduleId, tools) + setLinkedAgentTools(scope, moduleId, tools, agentRef) } } @@ -152,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number { // resource is missing or inaccessible so a broken link never stalls the flow load. export async function resolveLinkedAgentTools( agentRef: string, - workspace?: string + workspace: string | undefined, + withDraft: boolean ): Promise { const ws = workspace ?? get(workspaceStore) if (!ws) return [] - const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') + const path = normalizeAgentRef(agentRef) try { + if (withDraft) { + try { + const { response, draft } = await fetchAgentWithDraft(path, ws) + const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined + return (value?.tools ?? []) as AgentTool[] + } catch { + // The draft read failed for any reason. This is a display, not a run, so fall through to + // the deployed tools rather than showing an agent with none: an empty node list reads as + // "the agent lost its tools" instead of "we could not reach the server". The paths that + // act on a draft — the previews and the deploy dialog — surface the failure instead. + // Not rethrowing anything here: the outer catch turns every throw into `[]`, so a + // rethrow would skip the very fallback this exists for. + } + } const res = await ResourceService.getResource({ workspace: ws, path }) return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[] } catch { diff --git a/frontend/src/lib/components/flows/flowStateUtils.svelte.ts b/frontend/src/lib/components/flows/flowStateUtils.svelte.ts index 242455f557..20bd675700 100644 --- a/frontend/src/lib/components/flows/flowStateUtils.svelte.ts +++ b/frontend/src/lib/components/flows/flowStateUtils.svelte.ts @@ -1,3 +1,4 @@ +import { DEFAULT_AGENT_MEMORY } from './agentFormFields' import type { Schema } from '$lib/common' import { ScriptService, @@ -22,6 +23,19 @@ import { findNextAvailablePath } from '$lib/path' import type { ExtendedOpenFlow } from './types' import { emptySchema, type StateStore } from '$lib/utils' import { loadStoredConfig } from '../aiProviderStorage' +import { copilotInfo } from '$lib/aiStore' +import { stepSettingDefaults } from './flowStepSettings' +import type { InlineScript, InsertKind } from '$lib/components/graph/graphBuilder.svelte' +import { + agentToolToFlowModule, + createAiAgentTool, + createMcpTool, + createWebsearchTool, + newFlowModuleAgentTool, + SPECIAL_TOOL_KINDS, + type AgentTool, + type SpecialToolKind +} from './agentToolUtils' export async function loadFlowModuleState( flowModule: FlowModule, @@ -184,13 +198,14 @@ export async function createBranchAll(id: string): Promise<[FlowModule, FlowModu export async function createAiAgent( id: string, - agentPath?: string + agentPath?: string, + chatInputEnabled = false ): Promise<[FlowModule, FlowModuleState]> { const storedConfig = loadStoredConfig() const providerValue = storedConfig ?? { kind: 'openai', resource: '', model: '' } // A step linked to a saved agent reads its brain and tools from the resource, so it carries only - // the flow-local inputs: seeding `provider`/`output_type` would leave transforms it never reads. + // the flow-local inputs: seeding `provider` would leave a transform it never reads. const aiAgentFlowModules: FlowModule = { id, value: { @@ -202,7 +217,15 @@ export async function createAiAgent( ? {} : { provider: { type: 'static', value: providerValue }, - output_type: { type: 'static', value: 'text' } + // A chat agent answers a conversation, so it remembers it from the start. + ...(chatInputEnabled + ? { + memory: { + type: 'static' as const, + value: structuredClone(DEFAULT_AGENT_MEMORY) + } + } + : {}) }), user_message: { type: 'static', value: undefined } } @@ -442,3 +465,161 @@ export async function insertNewFailureModule( flowStateStore.val[module.id] = state } + +/** Create a new FlowModule without inserting it into any array. */ +export async function createNewModule( + flowStore: StateStore, + flowStateStore: StateStore, + kind: InsertKind, + wsScript?: { path: string; summary: string; hash: string | undefined }, + wsFlow?: { path: string; summary: string }, + inlineScript?: InlineScript, + agentPath?: string, + // The acting workspace when the flow editor runs in an AI session; else the nav workspace. + workspace?: string, + disableAi?: boolean +): Promise { + let module = emptyModule(flowStateStore.val, flowStore.val, kind == 'flow') + let state = emptyFlowModuleState() + flowStateStore.val[module.id] = state + if (wsFlow) { + ;[module, state] = await pickFlow(wsFlow.path, wsFlow.summary, module.id, workspace) + } else if (wsScript) { + ;[module, state] = await pickScript( + wsScript.path, + wsScript.summary, + module.id, + wsScript.hash, + kind, + workspace + ) + } else if (kind == 'forloop') { + ;[module, state] = await createLoop(module.id, !disableAi && get(copilotInfo).enabled) + } else if (kind == 'whileloop') { + ;[module, state] = await createWhileLoop(module.id) + } else if (kind == 'branchone') { + ;[module, state] = await createBranches(module.id) + } else if (kind == 'branchall') { + ;[module, state] = await createBranchAll(module.id) + } else if (kind == 'aiagent') { + ;[module, state] = await createAiAgent( + module.id, + agentPath, + flowStore.val.value?.chat_input_enabled ?? false + ) + } else if (inlineScript) { + const { language, kind, subkind, summary } = inlineScript + ;[module, state] = await createInlineScriptModule(language, kind, subkind, module.id, summary) + flowStateStore.val[module.id] = state + if (kind == 'trigger') { + module.summary = 'Trigger' + } else if (kind == 'approval') { + module.summary = 'Approval' + } + } + flowStateStore.val[module.id] = state + + if (kind == 'approval') { + module.suspend = stepSettingDefaults('suspend') + } else if (kind == 'trigger') { + module.stop_after_if = stepSettingDefaults('early-stop', 'trigger') + } else if (kind == 'end') { + module.summary = 'Terminate flow' + module.stop_after_if = stepSettingDefaults('early-stop', 'end') + } + + return module +} + +/** Create a module and splice it into `modules`, as a step or as one of an agent's tools. */ +export async function insertNewModuleAtIndex( + flowStore: StateStore, + flowStateStore: StateStore, + modules: FlowModule[] | AgentTool[], + index: number, + kind: InsertKind, + wsScript?: { path: string; summary: string; hash: string | undefined }, + wsFlow?: { path: string; summary: string }, + inlineScript?: InlineScript, + toolKind?: SpecialToolKind | 'flowmoduleTool', + // The acting workspace when the flow editor runs in an AI session; else the nav workspace. + workspace?: string, + disableAi?: boolean +): Promise { + const module = await createNewModule( + flowStore, + flowStateStore, + kind, + wsScript, + wsFlow, + inlineScript, + undefined, + workspace, + disableAi + ) + + if (!modules) return [module] + + if (toolKind === 'mcpTool') { + // Create MCP AgentTool + const mcpTool = createMcpTool(module.id) + ;(modules as AgentTool[]).splice(index, 0, mcpTool) + return modules as AgentTool[] + } else if (toolKind === 'websearchTool') { + // Create Websearch AgentTool + const websearchTool = createWebsearchTool(module.id) + ;(modules as AgentTool[]).splice(index, 0, websearchTool) + return modules as AgentTool[] + } else if (toolKind === 'aiAgentTool') { + // Create AI Agent tool (nested agent) + const aiAgentTool = createAiAgentTool(module.id) + flowStateStore.val[module.id] = await loadFlowModuleState( + agentToolToFlowModule(aiAgentTool), + workspace + ) + ;(modules as AgentTool[]).splice(index, 0, aiAgentTool) + return modules as AgentTool[] + } else if (toolKind === 'flowmoduleTool') { + const agentTool = newFlowModuleAgentTool(module) + ;(modules as AgentTool[]).splice(index, 0, agentTool) + return modules as AgentTool[] + } else { + // Standard FlowModule insertion + modules.splice(index, 0, module) + return modules + } +} + +/** + * Add one tool to an agent's `tools`, from the picker's detail, and return its id. Shared by the + * flow graph and the agent editor, which hold the same tools in different stores. + */ +export async function insertAgentTool( + flowStore: StateStore, + flowStateStore: StateStore, + agentValue: { tools?: AgentTool[] }, + detail: { kind: string; script?: any; flow?: any; inlineScript?: any }, + workspace?: string, + disableAi?: boolean +): Promise { + // `tools` is optional, so a module authored without it starts undefined here and would + // silently swallow its first tool. + agentValue.tools ??= [] + const tools = agentValue.tools + await insertNewModuleAtIndex( + flowStore, + flowStateStore, + tools, + tools.length, + detail.kind as InsertKind, + detail.script, + detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, + detail.inlineScript, + (SPECIAL_TOOL_KINDS as readonly string[]).includes(detail.kind) + ? (detail.kind as SpecialToolKind) + : 'flowmoduleTool', + workspace, + disableAi + ) + return tools[tools.length - 1]?.id +} diff --git a/frontend/src/lib/components/flows/flowStoreRefresh.svelte.ts b/frontend/src/lib/components/flows/flowStoreRefresh.svelte.ts deleted file mode 100644 index 2aa19f251a..0000000000 --- a/frontend/src/lib/components/flows/flowStoreRefresh.svelte.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { OpenFlow } from '$lib/gen' -import { refreshStateStore } from '$lib/svelte5Utils.svelte' -import type { StateStore } from '$lib/utils' -import { reanchorAgentEditsAcross } from './agentEditStore.svelte' - -// Use at content-preserving refresh sites (structural edits, schema/failure/mock changes): a bare -// refreshStateStore clone would break the agent edit marker's array identity. Content-changing -// replacements (undo, YAML/diff/AI apply, session restores) must stay bare so stale edit state -// keeps invalidating. -export function refreshFlowStateStore(flowStore: StateStore) { - reanchorAgentEditsAcross( - () => flowStore.val.value?.modules, - () => refreshStateStore(flowStore) - ) -} diff --git a/frontend/src/lib/components/flows/idUtils.ts b/frontend/src/lib/components/flows/idUtils.ts index db3e496c54..7aaecdd212 100644 --- a/frontend/src/lib/components/flows/idUtils.ts +++ b/frontend/src/lib/components/flows/idUtils.ts @@ -14,6 +14,9 @@ export const forbiddenIds: string[] = [ 'in', 'failure', 'preprocessor', + // The agent editor's synthetic root module. It shares one flow-state map with the agent's + // tools, so a tool of this id would take the root's slot and show the agent's own schema. + '__wm_agent_root', 'as', 'Input', 'Result', diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts new file mode 100644 index 0000000000..e9a84e771a --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -0,0 +1,203 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + inlineAgentDraft, + inlineAgentDrafts, + loadLinkedAgentDrafts, + repointLinkedAgent, + type LinkedAgentDraft +} from './linkedAgentDrafts' +import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' + +type AiAgentValue = Extract + +function linkedStep(input_transforms: Record): AiAgentValue { + return { + type: 'aiagent', + agent: 'f/team/support', + tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } }, + input_transforms + } as unknown as AiAgentValue +} + +describe('inlineAgentDraft', () => { + // The overlay order is the worker's: the resource brain first, the step's flow-local inputs on + // top. Reversing it would run the agent author's own `user_message` instead of the flow's. + it('keeps the step wired to the flow while the brain comes from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }), + { + provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }, + system_prompt: 'answer in french', + user_message: 'the default the agent carries', + tools: [{ id: 't1', summary: 'search' }] + } as any + ) + + expect(inlined.agent).toBeUndefined() + expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }]) + expect(inlined.input_transforms).toEqual({ + provider: { + type: 'static', + value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' } + }, + system_prompt: { type: 'static', value: 'answer in french' }, + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }) + // Host bindings are the step's, not the agent's, and the worker overlays them either way. + expect(inlined.tool_inputs).toEqual({ + t1: { query: { type: 'javascript', expr: 'flow_input.q' } } + }) + }) + + // A linked step carries only the flow-local inputs, but one persisted before linking existed can + // still hold stale brain transforms. They must not shadow the draft the test is meant to run. + it('drops brain transforms the step still carries', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + system_prompt: { type: 'static', value: 'stale' } + }), + { system_prompt: 'from the draft' } as any + ) + + expect(inlined.input_transforms).toEqual({ + system_prompt: { type: 'static', value: 'from the draft' }, + user_message: { type: 'static', value: 'hi' } + }) + }) + + // The worker never reads a history input from the resource, so a preview of the draft must not + // either: a draft carrying one would test against a memory the deployed step never sees. + it('never takes a history input from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + memory_id: { type: 'static', value: 'cust-1' } + }), + { + memory: { kind: 'window', context_length: 10 }, + memory_id: 'from-the-draft', + previous_messages: [{ role: 'user', content: 'from the draft' }] + } as any + ) + + expect(inlined.input_transforms).toEqual({ + memory: { type: 'static', value: { kind: 'window', context_length: 10 } }, + user_message: { type: 'static', value: 'hi' }, + memory_id: { type: 'static', value: 'cust-1' } + }) + }) +}) + +describe('inlineAgentDrafts', () => { + // The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the + // walk has to reach inside branches and loops. Miss either and every preview silently runs the + // deployed agent — the failure this whole path exists to prevent, and a silent one. + it('reaches a $res:-prefixed link nested in a branch', () => { + const value = { + modules: [ + { + id: 'b', + value: { + type: 'branchone', + default: [], + branches: [ + { + modules: [ + { + id: 'inner', + value: { + type: 'aiagent', + agent: '$res:f/team/support', + tools: [], + input_transforms: { user_message: { type: 'static', value: 'hi' } } + } + } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + const drafts = new Map([ + ['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft] + ]) + + const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0] + expect(inner.value.agent).toBeUndefined() + expect(inner.value.input_transforms.system_prompt).toEqual({ + type: 'static', + value: 'drafted' + }) + // The input the flow supplies survives the rewrite. + expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' }) + }) +}) + +describe('repointLinkedAgent', () => { + // A rename from the agent editor moves every step of the host flow onto the new path, nested ones + // included. Miss one and it silently stays linked to a path that no longer exists. + it('repoints linked steps at any depth and leaves other agents alone', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { + id: 'b', + value: { + type: 'branchall', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { id: 'd', value: { type: 'aiagent', agent: 'f/team/other', tools: [] } } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + expect(repointLinkedAgent(value, 'f/team/support', 'f/team/helpdesk')).toEqual(['a', 'c']) + const branch = (value.modules[1].value as any).branches[0].modules + expect((value.modules[0].value as any).agent).toBe('f/team/helpdesk') + expect(branch[0].value.agent).toBe('f/team/helpdesk') + expect(branch[1].value.agent).toBe('f/team/other') + }) +}) + +// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is +// an outage, and answering "no draft" to one would silently test or deploy against the deployed +// agent while the editor shows the draft. +describe('loadLinkedAgentDrafts error handling', () => { + function failWith(status: number | undefined) { + return async () => { + const err: Error & { status?: number } = new Error('boom') + err.status = status + throw err + } + } + + beforeEach(() => { + vi.restoreAllMocks() + }) + + it.each([401, 403, 404])('treats %i as no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map()) + }) + + it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow( + 'Could not load the agent f/team/support' + ) + }) +}) diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts new file mode 100644 index 0000000000..4a9aef24b1 --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -0,0 +1,253 @@ +import { + ResourceService, + type FlowModule, + type FlowValue, + type InputTransform, + type Resource +} from '$lib/gen' +import { UserDraft } from '$lib/userDraft.svelte' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' +import { dfs } from './dfs' +import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' +import { AGENT_HISTORY_KEYS } from './agentFormFields' +import type { AgentResourceState } from './agentDraft.svelte' +import type { AgentTool } from './agentToolUtils' + +/** A step names its agent bare or as `$res:`/`res://`; all three are the same agent, + * and a draft index has to answer for a lookup written any of those ways. Same normalization as + * `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */ +export function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as + * branches and loops, so a nested linked agent tool is included. */ +export function linkedAgentPaths(value: FlowValue | undefined): string[] { + if (!value?.modules) return [] + const paths = new Set() + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent) { + paths.add(normalizeAgentRef(v.agent)) + } + } + return [...paths] +} + +/** Point every step of this flow linked to `from` at `to`, for an agent renamed from inside it. + * Returns the ids of the steps it moved. */ +export function repointLinkedAgent( + value: FlowValue | undefined, + from: string, + to: string +): string[] { + if (!value?.modules) return [] + const moved: string[] = [] + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent === from) { + v.agent = to + moved.push(module.id) + } + } + return moved +} + +/** + * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then + * what a `get_draft` response carried. + * + * Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops + * the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is + * the sole answer. Read through `fetchAgentWithDraft`, which settles that row first. + */ +export function agentDraftState( + response: { draft?: unknown }, + path: string, + workspace: string | undefined +): AgentResourceState | undefined { + const live = UserDraft.get('resource', path, { workspace }) + return live ?? (response.draft as AgentResourceState | undefined) +} + +/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it + * would only repeat itself. */ +export class AgentDraftUnavailable extends Error {} + +/** + * An agent's resource together with the draft a run of it would use. + * + * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * closing the agent editor releases the in-memory cell without cancelling that pending POST — so + * testing or deploying right after closing would otherwise read a row the last edits have not + * reached yet. `flush` replays the parked save and is a no-op when there is none. + */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + const query = { workspace, itemKind: 'resource' as const, path } + await UserDraftDbSyncer.flush(query) + // `flush` resolves whether or not the save actually landed: `postSave` catches network and + // server errors into its failure map, and answers a conflicting write by parking a snapshot, + // returning normally in both cases. The row about to be read is then older than the edit still + // held in the browser, and nothing downstream could tell. Running that row is a test of the + // wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer + // edit with it. Neither is recoverable from here, so refuse the read. + const failure = UserDraftDbSyncer.getState(query).failureMessage + if (failure) { + throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`) + } + if (UserDraftDbSyncer.getConflict(query).conflict) { + throw new AgentDraftUnavailable( + `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` + ) + } + const response = await ResourceService.getResource({ workspace, path, getDraft: true }) + return { response, draft: agentDraftState(response, path, workspace) } +} + +/** One linked agent whose resource the user has an unsaved draft for. */ +export interface LinkedAgentDraft { + path: string + /** The draft's resource value: what a run of this agent would use. */ + args: AIAgentConfig + /** The whole draft row, as the resource editors write it — the deploy payload. */ + state: AgentResourceState + /** No deployed row at this path, so deploying has to create rather than update. */ + noDeployed: boolean + /** Of the deployed resource, for `agentDraftCanWrite`. */ + extraPerms: Record +} + +/** Whether `user` may write this agent's resource. Split from the load so that resolving the + * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it + * looks the user up once for every agent it lists. */ +export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { + return canWrite(draft.path, draft.extraPerms, user) +} + +/** A link that cannot resolve for the user rather than because something went wrong: the agent was + * deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and + * neither should stop the caller — the flow still tests and deploys, against the deployed agent. + * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy + * the wrong configuration, which is the whole thing this module exists to prevent. */ +export function isExpectedLinkFailure(err: unknown): boolean { + const status = (err as { status?: number } | null | undefined)?.status + return status === 401 || status === 403 || status === 404 +} + +/** + * The unsaved draft of every given `ai_agent` path, for the paths that have one. + * + * Throws when a path fails to load for any reason other than being missing or unreadable, so a + * caller cannot mistake an outage for an agent with nothing unsaved. + */ +export async function loadLinkedAgentDrafts( + paths: string[], + workspace: string | undefined +): Promise> { + const out = new Map() + if (!workspace || paths.length === 0) return out + await Promise.all( + paths.map(async (path) => { + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, workspace)) + } catch (err) { + if (isExpectedLinkFailure(err)) return + if (err instanceof AgentDraftUnavailable) throw err + throw new Error(`Could not load the agent ${path}: ${err}`) + } + if (!draft) return + out.set(path, { + path, + args: (draft.args ?? {}) as AIAgentConfig, + state: draft, + noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed), + extraPerms: response.extra_perms ?? {} + }) + }) + ) + return out +} + +/** + * Every argument a saved agent carries, as a static input transform. Not only the keys the agent + * form renders: a run reads them all, and an agent holding its own `user_message` answers with it + * when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on + * the module's `tools` key instead. + */ +export function agentArgsToTransforms(args: AIAgentConfig): Record { + const it: Record = {} + for (const [key, value] of Object.entries(args ?? {})) { + if (key === 'tools' || value === undefined) continue + it[key] = { type: 'static', value } as InputTransform + } + return it +} + +type AiAgentValue = Extract + +/** + * The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with + * the step's own flow-local inputs kept on top. + * + * The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole + * resource brain and only then writes the flow-local inputs (`user_message`, `user_attachments`, + * `enabled_tools`, `memory_id`, `previous_messages`) back from the step's own args. `tool_inputs` + * stays untouched — the worker overlays it onto the tools in both branches, so an inlined step + * keeps the host flow's tool bindings. The worker never reads a history input from the resource, + * so one a draft happens to carry is left out here too. + */ +export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue { + const { agent: _agent, ...rest } = value + const brain = agentArgsToTransforms(args) + for (const key of AGENT_HISTORY_KEYS) delete brain[key] + return { + ...rest, + tools: (args.tools ?? []) as AgentTool[], + input_transforms: { + ...brain, + ...flowLocalInputs(value.input_transforms as Record) + } + } as AiAgentValue +} + +/** + * Replace every linked agent step that has a draft with the draft's own configuration, so a preview + * runs what the agent editor is showing rather than the deployed resource. Returns a new value: the + * flow editor hands its live store object to previews. + */ +export function inlineAgentDrafts( + value: FlowValue, + drafts: Map +): FlowValue { + if (drafts.size === 0) return value + // JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which + // `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be + // posted as one — so the round trip loses nothing this preview would have carried. + const next = JSON.parse(JSON.stringify(value)) as FlowValue + for (const module of dfs(next.modules ?? [], (m) => m)) { + const v = module?.value as AiAgentValue | undefined + if (v?.type !== 'aiagent' || !v.agent) continue + const draft = drafts.get(normalizeAgentRef(v.agent)) + if (!draft) continue + module.value = inlineAgentDraft(v, draft.args) + } + return next +} + +/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a + * caller holding nothing but the value it is about to preview. */ +export async function withAgentDrafts( + value: FlowValue, + workspace: string | undefined +): Promise { + const paths = linkedAgentPaths(value) + if (paths.length === 0) return value + return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace)) +} diff --git a/frontend/src/lib/components/flows/linkedAgentToolsStore.svelte.ts b/frontend/src/lib/components/flows/linkedAgentToolsStore.svelte.ts index 503f30a38d..de879cc9c6 100644 --- a/frontend/src/lib/components/flows/linkedAgentToolsStore.svelte.ts +++ b/frontend/src/lib/components/flows/linkedAgentToolsStore.svelte.ts @@ -6,6 +6,28 @@ import type { AgentTool } from './agentToolUtils' // keeps flows shown at the same time (an editor and an embedded preview) from aliasing each other. let byScope = $state>>({}) +// Which agent each entry resolved from, so a write to one agent can be reflected on every step that +// links it rather than only the one that triggered the write. Plain, not reactive: it is read +// imperatively when reconciling a deploy, never rendered. +const agentRefByEntry = new Map() + +/** A step may name its agent bare or as `$res:`; both are the same agent, and the index has + * to answer for a lookup written either way. */ +function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +function entryKey(scope: string, moduleId: string): string { + return `${scope}\u0000${moduleId}` +} + +function forgetScopeRefs(scope: string) { + const prefix = `${scope}\u0000` + for (const key of agentRefByEntry.keys()) { + if (key.startsWith(prefix)) agentRefByEntry.delete(key) + } +} + // A long-lived tab would otherwise keep every flow it ever visited (raw tool script contents // included). Past this cap the least recently used scope is evicted, skipping any a mounted view // still holds. @@ -61,6 +83,7 @@ function evictOverCap(protect?: string) { } scopeOrder = scopeOrder.filter((s) => s !== victim) delete byScope[victim] + forgetScopeRefs(victim) } } @@ -85,7 +108,16 @@ export function linkedToolsScope( // track this to re-run when a link resolves after the initial render, e.g. right after linking. let version = $state(0) -export function setLinkedAgentTools(scope: string, moduleId: string, tools: AgentTool[]) { +export function setLinkedAgentTools( + scope: string, + moduleId: string, + tools: AgentTool[], + /** The agent these tools came from. Taken here rather than recorded separately so the index and + * the tools cannot drift: every writer has the ref in hand, and one that forgot to file it left + * a step invisible to `linkedModulesForAgent`. */ + agentRef: string +) { + agentRefByEntry.set(entryKey(scope, moduleId), normalizeAgentRef(agentRef)) // Publishers re-run and hand us a fresh-but-equal array each time; only mutate on a real change, // else the version bump would retrigger the graph recompute in a loop. if (deepEqual(byScope[scope]?.[moduleId], tools)) return @@ -94,12 +126,29 @@ export function setLinkedAgentTools(scope: string, moduleId: string, tools: Agen version++ } +/** Every module in this scope resolved from `agentRef`. A saved agent can be linked by more than + * one step, and all of them show tools that a write to it has just changed. */ +export function linkedModulesForAgent(scope: string, agentRef: string): string[] { + const prefix = `${scope}\u0000` + const wanted = normalizeAgentRef(agentRef) + const out: string[] = [] + for (const [key, ref] of agentRefByEntry) { + if (key.startsWith(prefix) && ref === wanted) out.push(key.slice(prefix.length)) + } + return out +} + /** Move one scope's resolutions into another: used when a rename moves readers to a new scope, * and to sweep republished data (keyed by the flow doc's path) into the live-edited scope. The * source bucket carries the newer resolution in both cases, so it wins the merge. */ export function migrateLinkedAgentToolsScope(oldScope: string, newScope: string) { if (oldScope === newScope || byScope[oldScope] === undefined) return byScope[newScope] = { ...(byScope[newScope] ?? {}), ...(byScope[oldScope] ?? {}) } + for (const moduleId of Object.keys(byScope[oldScope] ?? {})) { + const ref = agentRefByEntry.get(entryKey(oldScope, moduleId)) + if (ref !== undefined) agentRefByEntry.set(entryKey(newScope, moduleId), ref) + } + forgetScopeRefs(oldScope) delete byScope[oldScope] pendingMigration = newScope // Mark the new key most-recently-used but don't evict here: the next publish or release enforces @@ -109,6 +158,7 @@ export function migrateLinkedAgentToolsScope(oldScope: string, newScope: string) } export function clearLinkedAgentTools(scope: string, moduleId: string) { + agentRefByEntry.delete(entryKey(scope, moduleId)) if (byScope[scope]?.[moduleId] === undefined) return const rest = { ...byScope[scope] } delete rest[moduleId] diff --git a/frontend/src/lib/components/flows/linkedAgentToolsStore.test.ts b/frontend/src/lib/components/flows/linkedAgentToolsStore.test.ts index c8b96e8a7b..c310c80c24 100644 --- a/frontend/src/lib/components/flows/linkedAgentToolsStore.test.ts +++ b/frontend/src/lib/components/flows/linkedAgentToolsStore.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { clearLinkedAgentTools, getLinkedAgentTools, + linkedModulesForAgent, linkedToolsScope, migrateLinkedAgentToolsScope, releaseLinkedToolsScope, @@ -26,7 +27,7 @@ function freshScope(name = 'flow') { function fillPastCap(exclude: string) { for (let i = 0; i <= MAX_SCOPES; i++) { const filler = linkedToolsScope('filler', `${exclude}-${seq}-${i}`) - setLinkedAgentTools(filler, 'm', [tool(`f${i}`)]) + setLinkedAgentTools(filler, 'm', [tool(`f${i}`)], 'u/admin/a') } } @@ -34,15 +35,15 @@ describe('linkedAgentToolsStore', () => { it('keeps each scope module map separate', () => { const a = freshScope() const b = freshScope() - setLinkedAgentTools(a, 'step', [tool('x')]) - setLinkedAgentTools(b, 'step', [tool('y')]) + setLinkedAgentTools(a, 'step', [tool('x')], 'u/admin/a') + setLinkedAgentTools(b, 'step', [tool('y')], 'u/admin/a') expect(getLinkedAgentTools(a, 'step').map((t) => t.id)).toEqual(['x']) expect(getLinkedAgentTools(b, 'step').map((t) => t.id)).toEqual(['y']) }) it('evicts an unretained scope once past the cap', () => { const victim = freshScope() - setLinkedAgentTools(victim, 'step', [tool('x')]) + setLinkedAgentTools(victim, 'step', [tool('x')], 'u/admin/a') fillPastCap(victim) expect(getLinkedAgentTools(victim, 'step')).toEqual([]) }) @@ -50,7 +51,7 @@ describe('linkedAgentToolsStore', () => { // A run viewer holds one scope per mounted nested job; those must not evict what is on screen. it('never evicts a retained scope', () => { const held = freshScope() - setLinkedAgentTools(held, 'step', [tool('x')]) + setLinkedAgentTools(held, 'step', [tool('x')], 'u/admin/a') retainLinkedToolsScope(held) fillPastCap(held) expect(getLinkedAgentTools(held, 'step').map((t) => t.id)).toEqual(['x']) @@ -63,7 +64,7 @@ describe('linkedAgentToolsStore', () => { const held: string[] = [] for (let i = 0; i <= MAX_SCOPES; i++) { const scope = linkedToolsScope('retained', `${seq}-${i}`) - setLinkedAgentTools(scope, 'step', [tool(`t${i}`)]) + setLinkedAgentTools(scope, 'step', [tool(`t${i}`)], 'u/admin/a') retainLinkedToolsScope(scope) held.push(scope) } @@ -84,7 +85,7 @@ describe('linkedAgentToolsStore', () => { it('carries tools across a rename without evicting the new scope', () => { const from = freshScope('old') const to = freshScope('new') - setLinkedAgentTools(from, 'step', [tool('x')]) + setLinkedAgentTools(from, 'step', [tool('x')], 'u/admin/a') retainLinkedToolsScope(from) migrateLinkedAgentToolsScope(from, to) expect(getLinkedAgentTools(to, 'step').map((t) => t.id)).toEqual(['x']) @@ -98,13 +99,13 @@ describe('linkedAgentToolsStore', () => { const held: string[] = [] for (let i = 0; i < MAX_SCOPES; i++) { const scope = linkedToolsScope('rename-fill', `${seq}-${i}`) - setLinkedAgentTools(scope, 'step', [tool(`t${i}`)]) + setLinkedAgentTools(scope, 'step', [tool(`t${i}`)], 'u/admin/a') retainLinkedToolsScope(scope) held.push(scope) } const from = freshScope('before') const to = freshScope('after') - setLinkedAgentTools(from, 'step', [tool('x')]) + setLinkedAgentTools(from, 'step', [tool('x')], 'u/admin/a') retainLinkedToolsScope(from) migrateLinkedAgentToolsScope(from, to) @@ -121,10 +122,39 @@ describe('linkedAgentToolsStore', () => { it('clears one module without disturbing its siblings', () => { const scope = freshScope() - setLinkedAgentTools(scope, 'a', [tool('x')]) - setLinkedAgentTools(scope, 'b', [tool('y')]) + setLinkedAgentTools(scope, 'a', [tool('x')], 'u/admin/a') + setLinkedAgentTools(scope, 'b', [tool('y')], 'u/admin/a') clearLinkedAgentTools(scope, 'a') expect(getLinkedAgentTools(scope, 'a')).toEqual([]) expect(getLinkedAgentTools(scope, 'b').map((t) => t.id)).toEqual(['y']) }) + + // A deploy has to refresh every step showing the agent it wrote, so this index is what makes the + // difference between one step's nodes updating and all of them. + describe('linkedModulesForAgent', () => { + it('finds every module linked to one agent, and no others', () => { + const scope = freshScope() + setLinkedAgentTools(scope, 'a', [tool('x')], 'u/admin/one') + setLinkedAgentTools(scope, 'b', [tool('y')], 'u/admin/one') + setLinkedAgentTools(scope, 'c', [tool('z')], 'u/admin/other') + expect(linkedModulesForAgent(scope, 'u/admin/one').sort()).toEqual(['a', 'b']) + expect(linkedModulesForAgent(scope, 'u/admin/other')).toEqual(['c']) + }) + + it('reads a $res-prefixed link and a bare path as the same agent', () => { + const scope = freshScope() + setLinkedAgentTools(scope, 'a', [tool('x')], '$res:u/admin/one') + expect(linkedModulesForAgent(scope, 'u/admin/one')).toEqual(['a']) + }) + + it('drops a module that was cleared, and stays scoped', () => { + const scope = freshScope() + const other = freshScope() + setLinkedAgentTools(scope, 'a', [tool('x')], 'u/admin/one') + setLinkedAgentTools(other, 'a', [tool('x')], 'u/admin/one') + clearLinkedAgentTools(scope, 'a') + expect(linkedModulesForAgent(scope, 'u/admin/one')).toEqual([]) + expect(linkedModulesForAgent(other, 'u/admin/one')).toEqual(['a']) + }) + }) }) diff --git a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts index e85055df5b..29207f58f7 100644 --- a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts +++ b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts @@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({ isFlowModuleTool: () => false, agentToolToFlowModule: (t: unknown) => t })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } })) +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } +})) import { claimLinkedToolsFetch, @@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => { ) .mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never) - const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step') + const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false) release?.({ value: { tools: [tool('old')] } }) await stale @@ -56,8 +58,8 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - setLinkedAgentTools(scope, 'step', [tool('kept')]) + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a') invalidateLinkedToolsFetches(scope) release?.({ value: { tools: [tool('stale')] } }) @@ -73,9 +75,9 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) claimLinkedToolsFetch(scope, 'step') - setLinkedAgentTools(scope, 'step', [tool('direct')]) + setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a') release?.({ value: { tools: [tool('stale')] } }) await inFlight diff --git a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte index 9e6a18d82c..d54f8fddb1 100644 --- a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte @@ -1,5 +1,6 @@ +
    dispatch('close')} {disableAi} on:insert @@ -229,7 +309,10 @@ selected={selectedKind === 'aiagent'} onSelect={() => { selectedKind = 'aiagent' + selectedByKeyboard = 0 loadSavedAgents() + // Clicking leaves focus on this button, where Enter would only re-select it. + stepGen?.focus() }} /> {/if} @@ -239,6 +322,8 @@ selected={selectedKind === 'aisandbox'} onSelect={() => { selectedKind = 'aisandbox' + selectedByKeyboard = 0 + stepGen?.focus() }} /> {/if} @@ -247,19 +332,25 @@ {/if} {#if selectedKind === 'aiagent'} -
    +
    {#if savedAgentsLoading}
    @@ -267,20 +358,23 @@
    {:else if filteredAgents.length > 0}
    Saved agents
    - {#each filteredAgents as agent (agent.path)} + {#each filteredAgents as agent, i (agent.path)} {/each} {:else} @@ -295,17 +389,11 @@
    { - dispatch('close') - dispatch('new', { - kind: 'script', - inlineScript: { - language: 'bun', - kind: 'script', - subkind: 'claudesandbox' - } - }) - }} + neutral + returnIcon + selected={aiSelected === 0} + onSelect={newClaudeSandbox} + onHover={() => (selectedByKeyboard = 0)} />
    {:else} diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 234f053f7b..15177d6cd6 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubApps } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubApps() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 1f3c659bf8..8c07ad1506 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubFlows } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubFlows() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 207e8612e9..2e63266a58 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -4,12 +4,21 @@ import { capitalize } from '$lib/utils' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' + import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import ListFilters from '$lib/components/home/ListFilters.svelte' - import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' + import { ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import { disableHubStore } from '$lib/stores' import { logHubScriptPick } from '$lib/utils/featureUsage' + import { + alphabetical, + byPopularity, + localCountsByIntegration + } from '$lib/components/pickerPopularity' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { kind?: HubScriptKind & string @@ -46,19 +55,25 @@ }[] = $state([]) let allApps: string[] = $state([]) + let popularity: (a: string, b: string) => number = $state(alphabetical) let apps: string[] = $derived.by(() => - filter.length > 0 ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() : allApps + filter.length > 0 + ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity) + : allApps ) async function getAllApps(filterKind: typeof kind) { if ($disableHubStore) return try { hubNotAvailable = false - allApps = ( - await IntegrationService.listHubIntegrations({ - kind: filterKind - }) - ).map((x) => x.name) + // Independent reads, so they share one round trip before first paint. + const [integrations, local] = await Promise.all([ + listHubIntegrationsShared(filterKind), + $operatingWorkspace ? localCountsByIntegration($operatingWorkspace) : {} + ]) + const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) + popularity = byPopularity(hubPicks, local) + allApps = integrations.map((x) => x.name).sort(popularity) } catch (err) { console.error('Hub is not available') allApps = [] @@ -154,72 +169,73 @@ {#if $disableHubStore} {:else} -
    - {@render children?.()} -
    - - {#if loading} - - {/if} -
    -
    - -{#if hubNotAvailable} - - Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. - -{:else if (items.length > 0 && apps.length > 0) || !loading} - - {#if items.length == 0} - - {:else} -
      - {#each items as item (item.path)} -
    • - -
    • - {/each} -
    - {/if} - {#if items.length == 20} -
    - There are more items than being displayed. Refine your search. +
    + {@render children?.()} +
    + + {#if loading} + + {/if}
    +
    + + {#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the + Hub in the
    instance settings. + + {:else if (items.length > 0 && apps.length > 0) || !loading} + + {#if items.length == 0} + + {:else} +
      + {#each items as item (item.path)} +
    • + +
    • + {/each} +
    + {/if} + {#if items.length == 20} +
    + There are more items than being displayed. Refine your search. +
    + {/if} + {:else} + {#each Array(10).fill(0) as _} + + {/each} {/if} -{:else} - {#each Array(10).fill(0) as _} - - {/each} -{/if} {/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 5204892051..6648c622c0 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -1,9 +1,6 @@ -
    - - {selection.size} selected - -
    - - - {#if targets('archive').length > 0 || targets('unarchive').length === 0} + + {items.length} selected + +
    - {/if} - {#if targets('unarchive').length > 0} - - {/if} - open('discard'), - disabled: targets('discard').length === 0, - // Only while blocked: a disabled entry can't open its modal, so the reason - // has to live here — and an enabled one would render a pointless ⓘ. - tooltip: targets('discard').length === 0 ? actionTitle('discard') : undefined - }, - { - displayName: `Delete${countSuffix('delete')}`, - icon: Trash, - type: 'delete' as const, - action: () => open('delete'), - disabled: targets('delete').length === 0, - tooltip: targets('delete').length === 0 ? actionTitle('delete') : undefined - } - ]} - > - {#snippet buttonReplacement()} + + {#if targets('archive').length > 0 || targets('unarchive').length === 0} - {/snippet} - -
    - + {/if} + open('discard'), + disabled: targets('discard').length === 0, + // Only while blocked: a disabled entry can't open its modal, so the reason + // has to live here — and an enabled one would render a pointless ⓘ. + tooltip: targets('discard').length === 0 ? actionTitle('discard') : undefined + }, + { + displayName: `Delete${countSuffix('delete')}`, + icon: Trash, + type: 'delete' as const, + action: () => open('delete'), + disabled: targets('delete').length === 0, + tooltip: targets('delete').length === 0 ? actionTitle('delete') : undefined + } + ]} + > + {#snippet buttonReplacement()} + + {/snippet} + +
    +
    `${i.path} → ${movedPath(i, target)}`) + pendingItems.map((i) => `${sourcePath(i)} → ${movedPath(i, target)}`) )} {/if} {:else if pending === 'discard'} diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte index 4d8adb88e9..7c5bb4986e 100644 --- a/frontend/src/lib/components/home/CreateActionsMenu.svelte +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -15,6 +15,7 @@ Loader2, Workflow, Import, + Store, PanelLeftClose } from 'lucide-svelte' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' @@ -26,6 +27,25 @@ import { conditionalMelt, getLocalSetting, storeLocalSetting } from '$lib/utils' import { createDropdownMenu, melt } from '@melt-ui/svelte' import YAML from 'yaml' + import type { Snippet } from 'svelte' + import { logFeatureUsage } from '$lib/utils/featureUsage' + + interface Props { + /** Replaces the default `New` button, e.g. with an inline text link. */ + trigger?: Snippet + /** The node `trigger` renders: what the menu anchors to and what opens it. */ + triggerElement?: HTMLElement + /** Which entry point this menu hangs off, for telemetry. */ + source?: 'toolbar' | 'empty_state' + /** + * Opens the hub project picker. The menu only offers the entry; the picker and the + * import dialog belong to the host, which is the one place a single import modal can + * serve both this menu and the empty state's own link. + */ + onImportHubProject?: () => void + } + + let { trigger, triggerElement, source = 'toolbar', onImportHubProject }: Props = $props() type Variant = { label: string @@ -227,9 +247,17 @@ } } - let activeKey = $state(allOptions[0]?.key) - // every option's import action, surfaced together under the bottom "Import" submenu - const importActions: Extra[] = allOptions.flatMap((o) => o.extras ?? []) + // the doc panel only shows while an option is hovered or focused, so the menu opens compact + let activeKey: string | undefined = $state(undefined) + // every option's import action, surfaced together under the bottom "Import" submenu. + // The hub project leads and is separated below: the others each paste one artifact the + // user already holds, while this one brings a whole project in from somewhere else. + const importActions: Extra[] = $derived([ + ...(onImportHubProject + ? [{ label: 'Import a hub project', onSelect: onImportHubProject }] + : []), + ...allOptions.flatMap((o) => o.extras ?? []) + ]) // melt dropdown menu: arrow-key nav, typeahead, focus management and outside/escape // close all come for free; we only drive the doc panel off the highlighted item. @@ -306,7 +334,7 @@ // styling — melt element stores are callable on a node, exactly like `use:melt`. let triggerEl: HTMLButtonElement | HTMLAnchorElement | undefined = $state(undefined) $effect(() => { - const el = triggerEl + const el = triggerElement ?? triggerEl if (!el) return const applied = conditionalMelt(el, menuTrigger as any) as { destroy?: () => void @@ -314,6 +342,19 @@ return applied?.destroy }) + // Which entry point people actually create from: the toolbar button, or the inline + // link in the empty state. Only the open edge counts — melt writes the store on + // close and on every re-render of the menu. + let wasOpen = false + $effect(() => { + const isOpen = $open + if (isOpen && !wasOpen) { + logFeatureUsage('home', 'new_menu_open', { key: source }) + } + if (!isOpen) activeKey = undefined + wasOpen = isOpen + }) + const SHOW_DOC_SETTING = 'home_create_show_doc' let showDoc = $state(getLocalSetting(SHOW_DOC_SETTING) !== 'false') function setShowDoc(value: boolean) { @@ -321,8 +362,18 @@ // only persist the non-default (hidden) state, so a cleared key means "shown" storeLocalSetting(SHOW_DOC_SETTING, value ? undefined : 'false') } - let active = $derived(allOptions.find((o) => o.key === activeKey) ?? allOptions[0]) - let activeAc = $derived(accentClasses[active.accent]) + // The pointer crosses the gutter outside the menu on its way into the Workflow-as-Code + // submenu, so leaving the menu keeps the panel while that submenu is open; it clears + // once the submenu closes with neither pointer nor focus left on the menu. + let menuEl: HTMLDivElement | undefined = $state(undefined) + $effect(() => { + if ($wacSubOpen || !menuEl) return + if (!menuEl.matches(':hover') && !menuEl.contains(document.activeElement)) { + activeKey = undefined + } + }) + let active = $derived(allOptions.find((o) => o.key === activeKey)) + let activeAc = $derived(active ? accentClasses[active.accent] : undefined) // shared YAML/JSON import drawer, reused by every "Import …" extra let importDrawer: Drawer | undefined = $state(undefined) @@ -366,29 +417,49 @@ } -
    - - - {#if $open && active} -
    +
    + {#if trigger} + {@render trigger()} + {:else} + + {/if} +
    + +{#if $open} + +
    { + if (!$wacSubOpen) activeKey = undefined + }} + > +
    + {#if showDoc && active && activeAc}
    @@ -472,7 +543,7 @@
    {#each option.variants ?? [] as variant (variant.label)} {@const VariantIcon = variant.icon} @@ -505,6 +576,8 @@ + {#if onImportHubProject && i === 0} +
    + {/if} {/each}
    {/if} @@ -549,8 +629,8 @@ {/if}
    - {/if} -
    +
    +{/if} diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte new file mode 100644 index 0000000000..a8e4d8b973 --- /dev/null +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -0,0 +1,343 @@ + + + + +
    +
    + {#if showComposer && !collapsed} + {#if !disabled} + +
    + setCollapsed(true)} /> +
    + {/if} +
    +

    Build with AI

    + Beta +
    + +
    +
    + + {#if !value} + + + {/if} + +
    + +
    +
    + {#if disabled} + +
    +

    + {#if $aiUserDisabled} + Windmill AI is disabled in your account settings + {:else if freeTierExhausted} + You have used all of your free Windmill AI tokens + {:else} + No AI provider is configured + {/if} +

    +
    + {#if $aiUserDisabled} + + + {:else} + + {/if} + +
    +
    + {/if} +
    + {/if} + +
    + {#if showComposer && !collapsed} +
    + {#each examples as example (example.label)} + + {/each} +
    + {:else if showComposer} + + + {:else} +
    + {/if} + + +
    + + {#if !$userStore?.operator && HOME_SHOW_HUB} + + {/if} +
    +
    +
    +
    + + diff --git a/frontend/src/lib/components/home/HomeConnectDrawer.svelte b/frontend/src/lib/components/home/HomeConnectDrawer.svelte index e7e4a306d9..e08ac2ddea 100644 --- a/frontend/src/lib/components/home/HomeConnectDrawer.svelte +++ b/frontend/src/lib/components/home/HomeConnectDrawer.svelte @@ -5,10 +5,12 @@ import CopyableCodeBlock from '$lib/components/details/CopyableCodeBlock.svelte' import { Bot, ExternalLink, Terminal } from 'lucide-svelte' import { shell } from 'svelte-highlight/languages' + import { mcpTokenUrlDisabled } from '$lib/mcpAuth' type ConnectTab = 'cli' | 'mcp' let drawer: Drawer | undefined = $state() + let tokenUrlDisabled = $state(false) let selectedTab: ConnectTab = $state('cli') let openVersion = $state(0) @@ -24,6 +26,10 @@ wmill sync pull`) export function openDrawer(tab: ConnectTab = 'cli') { selectedTab = tab openVersion += 1 + // Falls back like CreateToken below, which shows the bare URL when the read fails. + void mcpTokenUrlDisabled() + .then((v) => (tokenUrlDisabled = v)) + .catch(() => (tokenUrlDisabled = true)) drawer?.openDrawer() } @@ -96,8 +102,13 @@ wmill sync pull`)

    MCP URL

    - Generate an MCP server URL for the current workspace and choose which - scripts, flows, and endpoints the client can access. + {#if tokenUrlDisabled} + The MCP server URL for the current workspace. Your client signs in to + Windmill to use it. + {:else} + Generate an MCP server URL for the current workspace and choose which + scripts, flows, and endpoints the client can access. + {/if}

    diff --git a/frontend/src/lib/components/home/HubProjectPickerModal.svelte b/frontend/src/lib/components/home/HubProjectPickerModal.svelte new file mode 100644 index 0000000000..6a353ea79d --- /dev/null +++ b/frontend/src/lib/components/home/HubProjectPickerModal.svelte @@ -0,0 +1,48 @@ + + + + + +
    + +
    +
    diff --git a/frontend/src/lib/components/home/HubTemplatePicker.svelte b/frontend/src/lib/components/home/HubTemplatePicker.svelte new file mode 100644 index 0000000000..a67c4be6a3 --- /dev/null +++ b/frontend/src/lib/components/home/HubTemplatePicker.svelte @@ -0,0 +1,181 @@ + + + +
    + +

    + Working projects from + + {hubHost} + + — imported as a folder in this workspace. +

    + +
    + + + {#snippet customRow({ + item + }: { + item: HubProjectPick & { pinned: boolean; leadsCatalogue: boolean } + })} + {@const Icon = hubAppIcon(item.iconApps[0] ?? '')} +
    + + + {/snippet} + + {#snippet empty()} +

    + {#if loadFailed} + Could not reach the hub. You can still browse its projects in a new tab. + {:else} + This hub has no projects yet. + {/if} +

    + {/snippet} + + + diff --git a/frontend/src/lib/components/home/ImportProjectModal.svelte b/frontend/src/lib/components/home/ImportProjectModal.svelte new file mode 100644 index 0000000000..d9ac59d3b3 --- /dev/null +++ b/frontend/src/lib/components/home/ImportProjectModal.svelte @@ -0,0 +1,363 @@ + + +{#snippet importPage()} +
    + {#if project} + + + {/if} + + (folder = f)} + onFinish={() => (setup.needed ? (onSetupStep = true) : finish('none'))} + onBack={onClose} + onExecution={(e) => (execution = e)} + resume={execution} + /> +
    +{/snippet} + +{#snippet setupPlaceholder()} + +
    + +
    +{/snippet} + +{#snippet setupPage()} +
    + finish('skipped', outstanding)} + onFinish={(checked) => finish(checked ? 'filled' : 'unchecked')} + onBack={execution ? () => (onSetupStep = false) : undefined} + /> +
    +{/snippet} + + + + {#if slug} + + (onSetupStep = s === 2)} + /> + + { + if (key === IMPORT_PAGE && execution) onSetupStep = false + else if (key === SETUP_PAGE && setup.needed) onSetupStep = true + }} + pages={[ + { key: IMPORT_PAGE, content: importPage }, + { key: SETUP_PAGE, content: setupPage, placeholder: setupPlaceholder } + ]} + /> + {/if} + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index d7717d1e04..f86ec9d5bf 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1,7 +1,7 @@ {#if Array.isArray(filtersAndSelected) && filtersAndSelected.length > 0} -
    +
    {#each displayedFilters as filter (filter)}
    - {#if resourceType} + {#if icon} + {@const Icon = icon} + + {:else if resourceType} {@const SvelteComponent = appIconComponent(filter)} {:else if filter.startsWith('u/')} @@ -114,7 +132,7 @@ {/if} - {filter} + {resourceType ? integrationDisplayName(filter) : filter} {#if filter === selectedFilter}✗{/if}
    @@ -123,7 +141,7 @@
    (expanded = !expanded)} diff --git a/frontend/src/lib/components/home/ListFiltersQuick.svelte b/frontend/src/lib/components/home/ListFiltersQuick.svelte index bca020b482..346fb93492 100644 --- a/frontend/src/lib/components/home/ListFiltersQuick.svelte +++ b/frontend/src/lib/components/home/ListFiltersQuick.svelte @@ -1,6 +1,7 @@ +{#snippet ownerNode(node: RootNode, loadRank: number, indent: number)} + +{/snippet} + {#if groupedItems === 'loading'}
    {:else}
    - {#each groupedItems.slice(0, nbDisplayed) as item, rootIndex ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} - {#if item} - + {#each rows.slice(0, nbDisplayed) as row (row.kind === 'otherUsers' ? 'other_users' : 'folderName' in row.node ? `f__${row.node.folderName}` : 'username' in row.node ? `u__${row.node.username}` : `i__${row.node.type}__${row.node.path}`)} + {#if row.kind === 'otherUsers'} + + + +
    +
    + +
    + Other users +
    + ({pluralize(otherUsers.length, 'user')}{otherUsersItemCount != undefined + ? ` · ${pluralize(otherUsersItemCount, 'item')}` + : ''}) +
    +
    +
    +
    + {#if otherUsersOpen} + + {#each otherUsers.slice(0, nbOtherUsersDisplayed) as user, i (user.username)} + {@render ownerNode(user, ownerRowCount + i, 1)} + {/each} + {#if nbOtherUsersDisplayed < otherUsers.length} +
    + + Showing {nbOtherUsersDisplayed} of {otherUsers.length} users + + +
    + {/if} + {/if} + {:else} + {@render ownerNode(row.node, row.loadRank, 0)} {/if} {/each} - {#if nbDisplayed < groupedItems.length || hasMoreServer} + {#if nbDisplayed < rows.length || hasMoreServer}
    - {#if nbDisplayed < groupedItems.length} - Showing {nbDisplayed} of {groupedItems.length} folders and users + {#if nbDisplayed < rows.length} + Showing {shownOwnerRowCount} of {ownerRowCount} folders and users {:else} @@ -231,12 +355,12 @@ unifiedSize="sm" variant="subtle" on:click={() => { - if (nbDisplayed < groupedItems.length) - nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, groupedItems.length) + if (nbDisplayed < rows.length) + nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, rows.length) else onLoadMore?.() }} > - {nbDisplayed < groupedItems.length ? 'Show more' : 'Load more'} + {nbDisplayed < rows.length ? 'Show more' : 'Load more'}
    {/if} diff --git a/frontend/src/lib/components/home/TutorialBanner.svelte b/frontend/src/lib/components/home/TutorialBanner.svelte deleted file mode 100644 index 14d8b750c1..0000000000 --- a/frontend/src/lib/components/home/TutorialBanner.svelte +++ /dev/null @@ -1,196 +0,0 @@ - - -{#if !isDismissed} -
    -
    - -
    -
    - {#if hasCompletedAny} - New tutorial available! - {:else} - Learn with interactive tutorials - {/if} -
    -
    - {#if hasCompletedAny} - Continue your learning journey and master new Windmill skills. - {:else} - Get started quickly with step-by-step guides on building flows, scripts, and more. - {/if} -
    -
    -
    -
    - - -
    -
    -{/if} diff --git a/frontend/src/lib/components/home/TutorialButton.svelte b/frontend/src/lib/components/home/TutorialButton.svelte deleted file mode 100644 index 6a31ae9569..0000000000 --- a/frontend/src/lib/components/home/TutorialButton.svelte +++ /dev/null @@ -1,124 +0,0 @@ - - - - diff --git a/frontend/src/lib/components/home/WorkspaceEmptyState.svelte b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte new file mode 100644 index 0000000000..fc36cc334b --- /dev/null +++ b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte @@ -0,0 +1,146 @@ + + +
    + {#each rowOpacities as opacity, i (i)} + + {/each} + + +
    + {#if archivedOnly} + + + Everything in this workspace is archived. + . + + {:else} + Your scripts, flows and apps will show up here. + {/if} + {#if canCreate} + + {#if !$disableHubStore} + + + e.detail && logFeatureUsage('home', 'template_picker_open', { key: 'empty_state' })} + > + {#snippet trigger()}Start from a template{/snippet} + {#snippet content({ close })} + { + close() + onPick(project) + }} + /> + {/snippet} + + or + {/if} + + {#snippet trigger()} + + . + {/snippet} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/home/bulkActions.test.ts b/frontend/src/lib/components/home/bulkActions.test.ts index 4710019671..d69c3b30ff 100644 --- a/frontend/src/lib/components/home/bulkActions.test.ts +++ b/frontend/src/lib/components/home/bulkActions.test.ts @@ -56,10 +56,12 @@ describe('blockedReason', () => { expect(blockedReason('unarchive', item({ archived: false }), admin)).toBeDefined() }) - it('routes a draft-only row to discard, never to move/archive/delete', () => { + it('lets a draft-only row move or discard, never archive/delete', () => { const draftOnly = item({ draftOnly: true, isDraft: true }) expect(blockedReason('discard', draftOnly, admin)).toBeUndefined() - expect(blockedReason('move', draftOnly, admin)).toBeDefined() + // Moving one rewrites its own draft row (DraftService.moveDraft) — there is + // no deployed path, but there is somewhere for it to go. + expect(blockedReason('move', draftOnly, admin)).toBeUndefined() expect(blockedReason('archive', draftOnly, admin)).toBeDefined() expect(blockedReason('delete', draftOnly, admin)).toBeDefined() }) @@ -75,4 +77,13 @@ describe('movedPath', () => { expect(movedPath(item({ path: 'f/alpha/sub/x' }), 'f/beta')).toBe('f/beta/sub/x') expect(movedPath(item({ path: 'u/ana/x' }), 'f/beta')).toBe('f/beta/x') }) + + it('names a draft-only row by what it displays, not the path it is parked at', () => { + const parked = item({ + draftOnly: true, + path: 'u/ana/draft_9f3c', + displayPath: 'u/ana/my_script' + }) + expect(movedPath(parked, 'f/beta')).toBe('f/beta/my_script') + }) }) diff --git a/frontend/src/lib/components/home/bulkActions.ts b/frontend/src/lib/components/home/bulkActions.ts index 161ddb0d72..da88e19121 100644 --- a/frontend/src/lib/components/home/bulkActions.ts +++ b/frontend/src/lib/components/home/bulkActions.ts @@ -8,11 +8,21 @@ * addresses the deployed row. A draft-only item is therefore not deletable — * there is nothing deployed at its path. */ -import { AppService, FlowService, ScriptService } from '$lib/gen' +import { AppService, DraftService, FlowService, ScriptService } from '$lib/gen' +import type { UserDraftItemKind } from '$lib/gen' import { updateItemPathAndSummary } from '$lib/components/moveRenameManager' import { discardDraft } from '$lib/utils_draft_deploy' import type { BulkItem } from './homeSelection.svelte' +/** The draft overlay is the one place a raw app is its own kind. Narrowed to the + * four kinds `moveDraft` accepts — a `BulkItem` is never anything else, and + * saying so lets the compiler check that rather than trusting it. */ +function draftKind( + item: BulkItem +): Extract { + return item.kind === 'app' && item.rawApp ? 'raw_app' : item.kind +} + export type BulkAction = 'move' | 'archive' | 'unarchive' | 'delete' | 'discard' export type BulkContext = { @@ -32,7 +42,6 @@ export function blockedReason( const notOwner = 'you are not an owner of this path' switch (action) { case 'move': - if (item.draftOnly) return 'a draft-only item has no deployed path to move' if (item.archived) return 'archived items cannot be moved' if (!item.owner) return notOwner if (!item.canWrite) return 'you do not have write permission on this path' @@ -64,17 +73,35 @@ export function eligible(action: BulkAction, items: BulkItem[], ctx: BulkContext return items.filter((i) => blockedReason(action, i, ctx) == undefined) } +/** The path a move reads from. A draft-only item is parked at a generated + * storage path but named by what was typed in the editor, and it is that name + * the user expects to find under the target. */ +export function sourcePath(item: BulkItem): string { + return item.draftOnly ? item.displayPath : item.path +} + /** Where an item lands under `target` (`f/` or `u/`): everything * below its own owner prefix is preserved, so nested paths keep their shape. */ export function movedPath(item: BulkItem, target: string): string { - const rest = item.path.split('/').slice(2).join('/') + const rest = sourcePath(item).split('/').slice(2).join('/') return `${target}/${rest}` } async function moveItem(ctx: BulkContext, item: BulkItem, target: string): Promise { const newPath = movedPath(item, target) // Re-saving a script at its current path would mint a pointless new version. - if (newPath === item.path) return + if (newPath === sourcePath(item)) return + if (item.draftOnly) { + // Nothing is deployed at this path, so there is no deploy to re-run: the + // item IS its draft row, and moving it rewrites that row. + await DraftService.moveDraft({ + workspace: ctx.workspace, + kind: draftKind(item), + path: item.path, + requestBody: { new_path: newPath } + }) + return + } await updateItemPathAndSummary({ workspace: ctx.workspace, kind: item.kind, @@ -117,10 +144,15 @@ async function deleteItem(ctx: BulkContext, item: BulkItem): Promise { } async function discardItemDraft(ctx: BulkContext, item: BulkItem): Promise { - // The draft overlay is the one place a raw app is its own kind. - const kind = item.kind === 'app' && item.rawApp ? 'raw_app' : item.kind // invalidate=false: the caller refreshes the draft list once for the batch. - const res = await discardDraft(kind, item.path, ctx.workspace, item.draftOnly, false, false) + const res = await discardDraft( + draftKind(item), + item.path, + ctx.workspace, + item.draftOnly, + false, + false + ) if (!res.success) throw new Error(res.error ?? 'discard failed') } diff --git a/frontend/src/lib/components/home/homeSelection.dom.test.ts b/frontend/src/lib/components/home/homeSelection.dom.test.ts new file mode 100644 index 0000000000..1e325fa3bd --- /dev/null +++ b/frontend/src/lib/components/home/homeSelection.dom.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { HomeSelection, type BulkItem } from './homeSelection.svelte' + +// A shift-click range resolves its ends through the rendered rows, so these need a +// document: the keys come back from `[data-row-selection-key]` in visual order. +function row(key: string): BulkItem { + return { + key, + kind: 'script', + path: key.slice('script/'.length), + displayPath: key.slice('script/'.length), + summary: '', + canWrite: true, + owner: true, + archived: false, + draftOnly: false, + isDraft: false, + rawApp: false + } +} + +const a = row('script/f/a/one') +const b = row('script/f/a/two') +const c = row('script/f/a/three') + +function rendered(...items: BulkItem[]): HomeSelection { + document.body.innerHTML = items + .map((i) => `
    `) + .join('') + const s = new HomeSelection() + s.available = true + for (const i of items) s.register(i) + return s +} + +describe('HomeSelection shift-range', () => { + beforeEach(() => { + document.body.innerHTML = '' + }) + + it('selects the span between the anchor and the clicked row', () => { + const s = rendered(a, b, c) + s.toggle(a) + + s.toggle(c, true) + + expect(s.items.map((i) => i.key)).toEqual([a.key, b.key, c.key]) + }) + + it('does not reach back into a selection the user already emptied', () => { + const s = rendered(a, b, c) + s.toggle(a) + s.toggle(a) + + s.toggle(c, true) + + expect(s.items.map((i) => i.key)).toEqual([c.key]) + }) +}) diff --git a/frontend/src/lib/components/home/homeSelection.svelte.ts b/frontend/src/lib/components/home/homeSelection.svelte.ts index fa755aabb3..97e2acc9d7 100644 --- a/frontend/src/lib/components/home/homeSelection.svelte.ts +++ b/frontend/src/lib/components/home/homeSelection.svelte.ts @@ -77,21 +77,16 @@ export class HomeSelection { /** The page offers multi-selection at all (never to an operator, and not on * the embedded read-only variants of the list). */ available = $state(false) - /** Selection mode is on even with nothing selected yet — entered from the - * toolbar, so every row reveals its checkbox before the first pick. */ - private explicit = $state(false) private selected = new SvelteMap() /** Every rendered selectable row, so a shift-click range can resolve the keys * between the anchor and the clicked row back to items. */ private registry = new SvelteMap() private anchor: string | undefined = undefined + /** Selection mode lasts exactly as long as something is selected: unticking + * the last row drops back to the normal list. */ get active(): boolean { - return this.available && (this.explicit || this.selected.size > 0) - } - - get size(): number { - return this.selected.size + return this.available && this.selected.size > 0 } get items(): BulkItem[] { @@ -133,12 +128,7 @@ export class HomeSelection { this.registry.delete(key) } - enter(): void { - this.explicit = true - } - exit(): void { - this.explicit = false this.selected.clear() this.anchor = undefined } @@ -155,13 +145,15 @@ export class HomeSelection { } toggle(item: BulkItem, range = false): void { - this.explicit = true if (range && this.anchor != undefined && this.anchor !== item.key) { if (this.selectRange(this.anchor, item.key)) return } if (this.selected.has(item.key)) this.selected.delete(item.key) else this.selected.set(item.key, item) - this.anchor = item.key + // Unticking the last row ends selection mode, so the row it was on is not an + // anchor any more: the next shift-click would otherwise reach back and select + // everything between it and the row the user just started from. + this.anchor = this.selected.size === 0 ? undefined : item.key } /** Visual order is read back from the DOM: the tree nests rows and pages them diff --git a/frontend/src/lib/components/icons/BRAND_COLORS.md b/frontend/src/lib/components/icons/BRAND_COLORS.md index 1cff77a3d3..11d4df5c3b 100644 --- a/frontend/src/lib/components/icons/BRAND_COLORS.md +++ b/frontend/src/lib/components/icons/BRAND_COLORS.md @@ -179,6 +179,7 @@ repeatedly — check the brand's own page. | `KeycloakIcon` | `keycloak` | fixed | #00B8E3 | #00B8E3 | 8.18 | 10.65 | keycloak.org's own mark, https://www.keycloak.org/resources/images/icon.svg (cyan #00B8E3/#33C6E9/#008AAA over greys #4D4D4D–#EDEDED, single theme) | | `KlaviyoIcon` | `klaviyo` | pair | #1D1E20 | #FFFFFF | 16.14 | 12.47 | klaviyo.com --color-core-charcoal; the flag mark is the standalone logomark the site header collapses to, and the shape of klaviyo.com/icons/icon-512x512.png | | `KoboToolboxIcon` | `kobotoolbox` | fixed | #2095F3 | #2095F3 | 3.05 | 3.95 | the kobotoolbox.org header logo and $kobo-blue in kobotoolbox/kpi jsapp/scss/colors.scss | +| `KubernetesIcon` | — | fixed | #326CE5 | #326CE5 | 4.61 | 2.62 | the CNCF artwork repo (github.com/cncf/artwork/projects/kubernetes/icon/color/kubernetes-icon-color.svg, CC-BY-4.0) | | `KustomerIcon` | `kustomer` | fixed | #FBEC2A | #FBEC2A | 14.08 | 12.47 | kustomer.com/images/kustomer/Kusty.svg | | `LangfuseIcon` | `langfuse` | fixed | #FF5D5F | #FF5D5F | 2.91 | 4.47 | langfuse.com/brand "Icon - Color (SVG)", used unmodified | | `LessIcon` | — | pair | #274F82 | #FFFFFF | 8.04 | 12.47 | github.com/less/logo (MIT) | @@ -443,6 +444,7 @@ Constraints that would otherwise be broken by a well-meaning change. - **JsonIcon** — JSON itself has no brand owner or published colours — json.org states none — so this is a Material palette pick, not a brand colour. - **KanidmIcon** — Kanidm's artwork is CC-BY-NC-ND — no recolouring or other derivatives. - **KlaviyoIcon** — Klaviyo draws it in currentColor, hence the white swap on dark. +- **KubernetesIcon** — The Linux Foundation trademark guidelines allow only the published variants (colour, all-blue, black, white); the colour mark's white knockouts are part of the artwork, not a theme swap. - **LangfuseIcon** — Langfuse's trademark terms forbid modifying the assets. - **LineIcon** — LINE forbids any change to the logo's colour, so there is no reversed variant. - **LinearIcon** — Guidelines ship a light/dark logomark pair and forbid altering the assets in any other way. @@ -571,7 +573,7 @@ Not brands. These inherit `currentColor` on purpose and must not be given a pair ## Coverage -- brand icons: **314**, of which **310** carry a recorded source +- brand icons: **315**, of which **311** carry a recorded source - per-theme pairs applied: **136** (5 of them by inversion or a two-SVG swap, see above) - concept icons: **34** - effectively invisible on light: **1** (AbstractApiIcon) diff --git a/frontend/src/lib/components/icons/KubernetesIcon.svelte b/frontend/src/lib/components/icons/KubernetesIcon.svelte new file mode 100644 index 0000000000..76279e002b --- /dev/null +++ b/frontend/src/lib/components/icons/KubernetesIcon.svelte @@ -0,0 +1,35 @@ + + + + + + + + + diff --git a/frontend/src/lib/components/inputTransformEnv.svelte.ts b/frontend/src/lib/components/inputTransformEnv.svelte.ts new file mode 100644 index 0000000000..9fe42c166e --- /dev/null +++ b/frontend/src/lib/components/inputTransformEnv.svelte.ts @@ -0,0 +1,56 @@ +import { CancelError, WorkspaceService, type LargeFileStorage } from '$lib/gen' +import { resource } from 'runed' + +/** + * Whether the workspace has large-file storage the upload endpoints can resolve. Every + * kind counts, not only S3: Azure Blob, Azure Workload Identity, S3 via AWS OIDC and GCS + * all go through the same object-store abstraction, so reading `s3_resource_path` alone + * calls a perfectly good workspace unconfigured. + */ +function storageConfigured(storage: LargeFileStorage | undefined): boolean { + if (!storage) return false + return ( + storage.type !== undefined || + storage.s3_resource_path !== undefined || + storage.azure_blob_resource_path !== undefined || + storage.gcs_resource_path !== undefined + ) +} + +/** + * Whether the workspace can store uploaded files; read `.current`. Assumed configured until + * this workspace's own answer lands, so "no storage" never flashes on navigation or shows + * merely because the fetch failed. + */ +export function useWorkspaceStorageConfigured(ws: () => string | undefined): { + readonly current: boolean +} { + const settings = resource(ws, async (ws, _previousWs, { onCleanup }) => { + if (!ws) return undefined + const req = WorkspaceService.getPublicSettings({ workspace: ws }) + // `resource` keeps whatever lands last: cancel a superseded request so a slow + // reply for a workspace we have left cannot overwrite the current one. + onCleanup(() => req.cancel()) + try { + return { ws, settings: await req } + } catch (err) { + if (!(err instanceof CancelError)) { + console.error('Failed to fetch workspace settings:', err) + } + return undefined + } + }) + + const configured = $derived.by(() => { + const loaded = settings.current + return loaded && loaded.ws === ws() + ? storageConfigured(loaded.settings.large_file_storage) + : true + }) + + return { + get current() { + return configured + } + } +} diff --git a/frontend/src/lib/components/instanceBanner.test.ts b/frontend/src/lib/components/instanceBanner.test.ts new file mode 100644 index 0000000000..bc93af18a1 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from 'vitest' +import { + INSTANCE_BANNER_MESSAGE_MAX_LEN, + isInstanceBannerVisible, + resolveInstanceBanner +} from './instanceBanner' + +const BANNER = { + enabled: true, + message: 'Scheduled maintenance on Saturday.', + severity: 'warning', + dismissible: true +} + +describe('resolveInstanceBanner', () => { + it('drops a link that is not an absolute http(s) URL', () => { + // Declarative instance config writes global_settings rows directly, so the API's + // scheme check is not the only thing standing between a stored value and an href. + for (const link of ['javascript:alert(1)', 'data:text/html,x', 'status.example.com', 123]) { + expect(resolveInstanceBanner({ ...BANNER, link })?.link).toBeUndefined() + } + expect(resolveInstanceBanner({ ...BANNER, link: 'https://status.example.com' })?.link).toBe( + 'https://status.example.com' + ) + }) + + it('keeps a message the backend accepted whole', () => { + // The backend caps at INSTANCE_BANNER_MESSAGE_MAX_LEN code points (`chars().count()`). + // Truncating with `slice` here would count UTF-16 units and halve an all-emoji message + // that passed validation, so the two sides must measure the same way. + const emoji = '\u{1F6A7}'.repeat(INSTANCE_BANNER_MESSAGE_MAX_LEN) + expect([...resolveInstanceBanner({ ...BANNER, message: emoji })!.message]).toHaveLength( + INSTANCE_BANNER_MESSAGE_MAX_LEN + ) + }) + + it('shows nothing when disabled or without a message', () => { + // An enabled banner with no message is a writable state (the backend accepts it so a + // half-typed announcement cannot fail an admin's whole settings save), so this is the + // only thing keeping it off everyone's screen. + expect(resolveInstanceBanner({ ...BANNER, enabled: false })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: ' ' })).toBeUndefined() + expect(resolveInstanceBanner({ ...BANNER, message: 42 })).toBeUndefined() + }) +}) + +describe('isInstanceBannerVisible', () => { + it('honours a dismissal only while the announcement is dismissible', () => { + const dismissible = resolveInstanceBanner(BANNER)! + expect(isInstanceBannerVisible(dismissible, dismissible.fingerprint)).toBe(false) + expect(isInstanceBannerVisible(dismissible, 'some other announcement')).toBe(true) + + // Escalating the same announcement to mandatory must reach the people who already + // dismissed it — the fingerprint does not change, so nothing else would bring it back. + const mandatory = resolveInstanceBanner({ ...BANNER, dismissible: false })! + expect(mandatory.fingerprint).toBe(dismissible.fingerprint) + expect(isInstanceBannerVisible(mandatory, mandatory.fingerprint)).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/instanceBanner.ts b/frontend/src/lib/components/instanceBanner.ts new file mode 100644 index 0000000000..f1693024b8 --- /dev/null +++ b/frontend/src/lib/components/instanceBanner.ts @@ -0,0 +1,129 @@ +import { SettingService } from '$lib/gen' + +/** Drives the banner palette and icon; the subset of `AlertType` that fits an announcement. */ +export type InstanceBannerSeverity = 'info' | 'warning' | 'error' + +/** Stored shape of the `instance_banner` global setting. Every field is optional: a + * stored value can predate a field this code knows about. */ +export interface InstanceBanner { + enabled?: boolean + message?: string + severity?: InstanceBannerSeverity + /** Whether a viewer may dismiss the banner for themselves. Absent means yes. */ + dismissible?: boolean + link?: string + link_label?: string +} + +export const INSTANCE_BANNER_SETTING = 'instance_banner' + +/** Mirror `INSTANCE_BANNER_MESSAGE_MAX_LEN` / `INSTANCE_BANNER_LINK_LABEL_MAX_LEN` in + * backend/windmill-common/src/global_settings.rs, which reject longer values at write time. */ +export const INSTANCE_BANNER_MESSAGE_MAX_LEN = 500 +export const INSTANCE_BANNER_LINK_LABEL_MAX_LEN = 60 + +export type ResolvedInstanceBanner = { + message: string + severity: InstanceBannerSeverity + dismissible: boolean + link?: string + linkLabel: string + /** Dismissal token: a viewer who dismissed one announcement sees the next one, + * because editing any displayed part of the banner changes this string. */ + fingerprint: string +} + +/** + * Read a stored banner field as a string. + * + * The setting is a raw `global_settings` row, so its shape is only ever as good as the + * writer that last touched it — and anything here that calls `.trim()` on a number throws, + * taking the whole settings form down with it. + */ +export function bannerString(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +/** + * Truncate to `max` code points, matching the backend's `chars().count()` cap. + * + * `String.slice` counts UTF-16 code units, so it would cut a 500-emoji message the backend + * accepted in half — and the two sides must agree on what "500 characters" means. + */ +function truncateChars(value: string, max: number): string { + const chars = [...value] + return chars.length > max ? chars.slice(0, max).join('') : value +} + +export function isHttpUrl(value: string): boolean { + try { + const url = new URL(value) + return url.protocol === 'http:' || url.protocol === 'https:' + } catch { + return false + } +} + +/** + * Turn the raw setting into what the banner renders, or `undefined` for "show nothing". + * + * The scheme check repeats the one the writers run on purpose. The link becomes the href of + * an anchor shown to every user of the instance, and this is the last place that can refuse + * it — a row predating the validator, or written straight to the table, reaches here having + * passed nothing. + */ +export function resolveInstanceBanner(raw: unknown): ResolvedInstanceBanner | undefined { + if (!raw || typeof raw !== 'object') return undefined + const banner = raw as InstanceBanner + const message = bannerString(banner.message).trim() + if (banner.enabled !== true || message === '') return undefined + + const severity: InstanceBannerSeverity = + banner.severity === 'warning' || banner.severity === 'error' ? banner.severity : 'info' + const rawLink = bannerString(banner.link).trim() + const link = isHttpUrl(rawLink) ? rawLink : undefined + const linkLabel = + truncateChars(bannerString(banner.link_label).trim(), INSTANCE_BANNER_LINK_LABEL_MAX_LEN) || + 'Learn more' + + return { + message: truncateChars(message, INSTANCE_BANNER_MESSAGE_MAX_LEN), + severity, + dismissible: banner.dismissible !== false, + link, + linkLabel, + fingerprint: JSON.stringify([message, severity, link ?? '', link ? linkLabel : '']) + } +} + +/** + * The reason the banner form cannot be saved, or `undefined` when it can. + * + * Shared with the setting's `isValid` so the Save button and the inline message agree: the + * backend refuses a bad link, and a category save fires its settings concurrently, so a Save + * that got this far would persist the other Core settings and fail only the banner. + */ +export function instanceBannerFormError(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const link = bannerString((value as InstanceBanner).link).trim() + return link !== '' && !isHttpUrl(link) ? 'Link must be an absolute http(s) URL' : undefined +} + +/** + * Whether a viewer holding `dismissedFingerprint` should see this announcement. + * + * A non-dismissible announcement ignores stored dismissals entirely: an admin escalating an + * existing notice to mandatory must reach the people who already dismissed it, and the + * fingerprint deliberately does not cover `dismissible`, so nothing else would bring it back. + */ +export function isInstanceBannerVisible( + banner: ResolvedInstanceBanner | undefined, + dismissedFingerprint: string +): banner is ResolvedInstanceBanner { + if (banner == undefined) return false + return !banner.dismissible || dismissedFingerprint !== banner.fingerprint +} + +export async function fetchInstanceBanner(): Promise { + return resolveInstanceBanner(await SettingService.getGlobal({ key: INSTANCE_BANNER_SETTING })) +} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 6c66af7d84..1fe45931a2 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -1,5 +1,8 @@ import type { ButtonType } from './common/button/model' +import { allowedOriginsSettingError } from './triggers/http/utils' import { z } from 'zod' +import { instanceBannerFormError } from './instanceBanner' +import { parseMaxTokenExpirationDays } from '$lib/tokenExpiration' import { writable } from 'svelte/store' /** @@ -30,6 +33,10 @@ export interface Setting { placeholder?: string cloudonly?: boolean ee_only?: string + /** Ceiling a `seconds` field enforces on a build without a license, when CE genuinely caps + * the value. Not implied by `ee_only`: a setting can be EE-badged because the feature it + * configures is EE while the value itself has the same range on either edition. */ + ceMaxSeconds?: number tooltip?: string key: string // If value is not specified for first element, it will automatcally use undefined @@ -64,6 +71,7 @@ export interface Setting { | 'webhook_base_url' | 'ws_connectivity' | 'retention_overrides' + | 'instance_banner' storage: SettingStorage advancedToggle?: { label: string @@ -262,6 +270,22 @@ export const settings: Record = { ee_only: '', hideInQuickSetup: true }, + { + label: 'HTTP route default allowed origins', + description: + 'Origins that HTTP routes allow to call them from a browser when the route sets none of its own. A route overrides this with its own list, and opts out entirely by setting its allowed origins to *. Leave unset for no instance-wide default, so every route is callable from any origin unless it restricts itself.', + key: 'http_route_default_allowed_origins', + fieldType: 'text', + placeholder: 'https://app.example.com, https://admin.example.com', + storage: 'setting', + error: + 'Each origin must be visible ASCII with no comma, and there can be at most 100 of them. null is not allowed, since every sandboxed iframe sends it.', + // The same check the API applies, so a value it would refuse cannot be + // saved here and then silently drop to no restriction at the next boot. + isValid: (value: unknown) => allowedOriginsSettingError(value) === undefined, + ee_only: '', + hideInQuickSetup: true + }, { label: 'Audit log retention (days)', key: 'audit_log_retention_days', @@ -271,6 +295,45 @@ export const settings: Record = { storage: 'setting', ee_only: '', hideInQuickSetup: true + }, + { + label: 'Maximum token expiration (days)', + key: 'max_token_expiration_days', + description: + 'Furthest ahead an API token a user creates can expire, in days. A token asking for longer, or for no expiration, is created with this expiration instead. Service accounts are exempt, so automation can keep longer-lived credentials. Leave empty to let users pick any expiration, including none.', + fieldType: 'number', + placeholder: 'no limit', + storage: 'setting', + hideInQuickSetup: true, + error: 'Must be a whole number of days, from 1 to 1,000,000', + // The server reads anything else as no ceiling at all. + isValid: (value: unknown) => + value === undefined || + value === null || + value === '' || + parseMaxTokenExpirationDays(value) !== undefined + }, + { + label: 'Disable token in MCP URLs', + description: + 'Reject the ?token= query parameter on the MCP endpoints, so MCP clients authenticate with an Authorization header or through the OAuth flow. A token in a URL is a credential that ends up in browser history, proxy logs and referrers. Existing MCP URLs carrying a token stop working. Servers and workers pick this up within a minute; dedicated MCP servers (MODE=mcp) apply it when they next restart.', + key: 'mcp_disable_token_query_param', + fieldType: 'boolean', + storage: 'setting', + hideInQuickSetup: true + }, + { + label: 'Announcement banner', + description: + 'Message shown above every page of the instance, for maintenance windows and incidents.', + key: 'instance_banner', + fieldType: 'instance_banner', + storage: 'setting', + // The banner only renders on EE, so the card is disabled without a license. + ee_only: '', + hideInQuickSetup: true, + // Gates Save. The card renders the specific message itself, so no `error` here. + isValid: (value: any) => instanceBannerFormError(value) == undefined } ], Jobs: [ @@ -283,6 +346,8 @@ export const settings: Record = { placeholder: '30', storage: 'setting', ee_only: 'You can only adjust this setting to above 30 days in the EE version', + // Mirrors CE_MAX_RETENTION_PERIOD_SECS, which the backend clamps to on write. + ceMaxSeconds: 60 * 60 * 24 * 30, cloudonly: false }, { @@ -525,6 +590,17 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'Back AI sessions up to the instance object storage', + description: + "Browsers back their AI sessions up to their workspace's object storage, encrypted with the workspace key. When this is on and instance object storage is configured, a workspace without object storage of its own uses the instance object storage instead, under the same encryption; configuring a storage for the workspace moves its backups there and deletes what it kept in the instance storage. On by default; turn off to keep the AI sessions of such workspaces in the browser only.", + key: 'ai_sessions_instance_storage_fallback', + fieldType: 'boolean', + defaultValue: () => true, + storage: 'setting', + ee_only: '', + hideInQuickSetup: true + }, { label: 'Store audit logs in object storage', description: @@ -656,6 +732,16 @@ export const settings: Record = { fieldType: 'text', placeholder: 'okta', storage: 'setting' + }, + { + label: 'SSO groups claim', + description: + 'Name of the SAML attribute or OIDC userinfo claim carrying the user\'s IdP groups ("http://schemas.microsoft.com/ws/2008/06/identity/claims/groups" on Entra SAML, "groups" for most OIDC providers). Its values must be the same group ids that SCIM stored as the instance groups\' external id (Entra emits object ids in both), since matching is by external id only. When set, every SSO login reconciles the user\'s membership in those SCIM-provisioned instance groups against the claim, so IdP group changes take effect at the next login instead of waiting for the SCIM push. Instance groups without an external id are never touched, and a login whose claim is absent or empty changes nothing. Leave empty to disable.', + key: 'sso_groups_claim', + fieldType: 'text', + placeholder: 'groups', + storage: 'setting', + ee_only: '' } ], 'DB Health': [], @@ -971,6 +1057,24 @@ export const settings: Record = { triggersRestart: true, defaultValue: () => ({ enabled: false, enabled_languages: [...OTEL_TRACING_PROXY_LANGUAGES] }) }, + { + label: 'HTTP Request Tracing retention in secs', + key: 'otel_traces_retention_secs', + description: + 'How long a captured HTTP request span is kept in the database, and therefore how far back the job details view can show a job its requests. Independent of the job retention period, so a span may outlive its job or be swept while the job remains. Defaults to 7 days. Leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + // Badged EE because only the EE proxy captures spans, but deliberately no + // `ceMaxSeconds`: a CE build still sweeps rows an EE-era instance left behind, and + // the backend accepts the same range on either edition. + ee_only: 'HTTP Request Tracing is an EE feature', + error: + 'HTTP Request Tracing retention must be between 1 second and 100 years, leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + }, { label: 'Prometheus', description: @@ -982,6 +1086,23 @@ export const settings: Record = { triggersRestart: true } ], + 'Service logs': [ + { + label: 'Retention in secs', + key: 'service_log_retention_secs', + description: + 'How long a service log is kept, across every copy of it: the entry in the database, the file on the disk of the process that wrote it, and — once instance object storage is configured and the indexer has ingested it — its line in the columnar store that search and the log viewer read. Search reaches back at most this far, and less when the indexer time window under Indexer is shorter. Defaults to 14 days. There is no keep-forever setting here — leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + error: + 'Service log retention must be between 1 second and 100 years — leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + } + ], + Indexer: [ { label: '', @@ -1173,6 +1294,12 @@ export const instanceSettingsNavigationGroups = [ aiDescription: 'Instance OTEL/Prometheus settings', isEE: true }, + { + id: 'service_logs', + label: 'Service logs', + aiId: 'instance-settings-service-logs', + aiDescription: 'Service log retention settings' + }, { id: 'indexer', label: 'Indexer', @@ -1256,6 +1383,7 @@ export const tabToCategoryMap: Record = { webhooks: 'Webhooks', otel_prom: 'OTEL/Prom', indexer: 'Indexer', + service_logs: 'Service logs', telemetry: 'Telemetry', secret_storage: 'Secret Storage', object_storage: 'Object Storage', @@ -1291,6 +1419,7 @@ export const categoryToTabMap: Record = { Webhooks: 'webhooks', 'OTEL/Prom': 'otel_prom', Indexer: 'indexer', + 'Service logs': 'service_logs', Telemetry: 'telemetry', 'Secret Storage': 'secret_storage', 'Object Storage': 'object_storage', diff --git a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte index da63e8d64a..acd18130da 100644 --- a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte @@ -283,7 +283,17 @@ placeholder: '12345', disabled: fieldsDisabled }} - bind:value={$values['github_enterprise_app'].app_id} + bind:value={ + () => $values['github_enterprise_app'].app_id, + (v) => { + // The backend expects app_id as a positive integer (i64). Reject + // fractional/out-of-range values instead of truncating them, and store + // undefined (never a string or 0) so the config omits the key when unset. + const n = typeof v === 'string' ? Number(v.trim() || NaN) : (v ?? NaN) + $values['github_enterprise_app'].app_id = + Number.isSafeInteger(n) && n > 0 ? n : undefined + } + } />
    diff --git a/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte new file mode 100644 index 0000000000..d55c2f9ce6 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/GuestActivityList.svelte @@ -0,0 +1,133 @@ + + + + +{#if !usage.available} +
    + + No guest can sign in here, whatever a workspace or an app says. Guests require a self-hosted + instance or a dedicated Windmill Cloud deployment. + +
    +{:else} +
    + {#key usage} + setInstanceSwitch(e.detail)} + options={{ + right: 'Allow guests on this instance', + rightTooltip: + 'Off, no guest can sign in anywhere, whatever a workspace or an app says, and sessions already issued stop on their next request.' + }} + /> + {/key} +
    + +
    + + {#if usage.metered} + Beyond the allowance, every four guests count as one seat{usage.guest_seats > 0 + ? `: ${usage.billable_guests} guests past it take ${usage.guest_seats} ${usage.guest_seats === 1 ? 'seat' : 'seats'} now` + : ''}. + {:else} + Beyond the allowance, new guests are refused until the count drops below it; an Enterprise + license meters them instead. + {/if} + +
    +{/if} + + onLoadMore()} +> + +
    + Email + Workspaces + First seen + Last seen + + + + {#each guests as guest, i (guest.email)} + + {guest.email} + {guest.workspaces.join(', ')} + {guest.first_seen} + {guest.last_seen} + + {/each} + + diff --git a/frontend/src/lib/components/instanceSettings/InstanceBannerSetting.svelte b/frontend/src/lib/components/instanceSettings/InstanceBannerSetting.svelte new file mode 100644 index 0000000000..293f567a0b --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/InstanceBannerSetting.svelte @@ -0,0 +1,159 @@ + + +
    + banner.enabled === true, (v) => ($values[INSTANCE_BANNER_SETTING].enabled = v) + } + options={{ right: 'Show the banner to every user' }} + /> + +
    + Message + bannerString(banner.message), + (v) => ($values[INSTANCE_BANNER_SETTING].message = String(v)) + } + /> +
    + +
    + Severity + bannerString(banner.severity) || 'info', + (v) => ($values[INSTANCE_BANNER_SETTING].severity = v) + } + > + {#snippet children({ item })} + {#each severities as severity (severity.value)} + + {/each} + {/snippet} + +
    + +
    + Link (optional) +
    + bannerString(banner.link), + (v) => ($values[INSTANCE_BANNER_SETTING].link = String(v)) + } + /> + bannerString(banner.link_label), + (v) => ($values[INSTANCE_BANNER_SETTING].link_label = String(v)) + } + /> +
    + {#if linkError} + {linkError} + {/if} +
    + + banner.dismissible !== false, (v) => ($values[INSTANCE_BANNER_SETTING].dismissible = v) + } + options={{ + right: 'Let users dismiss it', + rightTooltip: + 'Dismissal is remembered per browser and only for this exact announcement: editing the message, severity or link brings it back for everyone. Turning this off also shows it again to everyone who had dismissed it.' + }} + /> + +
    + Preview + {#if preview} + {@const palette = alertClasses[preview.severity]} + {@const Icon = alertIcons[preview.severity]} +
    + + {preview.message} + {#if preview.link} + {preview.linkLabel} + {/if} +
    + {:else} + + {banner.enabled === true + ? 'Nothing is shown until the message is filled in.' + : 'Nothing is shown while the banner is off.'} + + {/if} +
    +
    diff --git a/frontend/src/lib/components/job_args.test.ts b/frontend/src/lib/components/job_args.test.ts new file mode 100644 index 0000000000..cf5818ff0f --- /dev/null +++ b/frontend/src/lib/components/job_args.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from 'vitest' +import { + coerceArgsToSchema, + enforceDisabledDefaults, + redactFileArgs, + redactSecretArgs +} from './job_args' + +describe('coerceArgsToSchema', () => { + // A scalar widget renders its own reading of a wrong-typed value and never writes that + // reading back, so an untouched form submits something it never displayed: a number + // input paints `"12"` as a filled-looking 12, and a toggle shows `"false"` as on. + it('converts a value its widget would read, so the form shows what runs', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' }, + name: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: '12', flag: 'false', label: 3, name: 'ada' }, + schema + ) + expect(args).toEqual({ count: 12, flag: false, label: '3', name: 'ada' }) + expect(clearedKeys).toEqual([]) + }) + + // Cleared, not carried: the widget shows nothing for these, so nothing is what an + // untouched form should send. + it('empties a value with no reading in its declared type', () => { + const schema = { + properties: { + count: { type: 'number' }, + flag: { type: 'boolean' }, + label: { type: 'string' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { count: 'abc', flag: 'maybe', label: { a: 1 } }, + schema + ) + expect(args).toEqual({}) + expect(clearedKeys.sort()).toEqual(['count', 'flag', 'label']) + }) + + // A field the schema does not name is a field no run surface in the product draws, so a + // value under that name would reach the job without anyone having been able to see it. + // `constructor` is declared by every object through its prototype and by no schema. + it('drops arguments the schema does not declare, naming them', () => { + const kept = coerceArgsToSchema({ a: 'keep', b: 2, constructor: 'x' }, { + properties: { a: { type: 'string' } } + } as any) + expect(kept.args).toEqual({ a: 'keep' }) + expect(kept.undeclaredKeys).toEqual(['b', 'constructor']) + // Declaring nothing is declaring no arguments, which is what a `**kwargs` script and a + // schema that failed to infer both look like. + expect(coerceArgsToSchema({ a: 1 }, undefined).args).toEqual({}) + }) + + // Resolved by the job, so the declared type describes what it receives and never the + // string standing in for it. `Number('$var:…')` is NaN, so coercing would destroy it. + it('leaves a variable or resource reference in any slot', () => { + const schema = { + properties: { + size: { type: 'number' }, + on: { type: 'boolean' }, + db: { type: 'object', format: 'resource-postgresql' } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { size: '$var:u/admin/size', on: '$var:u/admin/on', db: '$res:u/admin/pg' }, + schema + ) + expect(args).toEqual({ + size: '$var:u/admin/size', + on: '$var:u/admin/on', + db: '$res:u/admin/pg' + }) + expect(clearedKeys).toEqual([]) + }) + + // Not merely unreadable: `MultiSelect` maps over the value as it renders, so anything + // else throws and takes the whole card down, Cancel with it. A reference is no + // exception — the widget draws before anything resolves — so this slot is the one + // place the reference rule above does not hold. + it('empties a non-array in a dyn-multiselect slot, reference included', () => { + const schema = { properties: { tags: { type: 'object', format: 'dynmultiselect-list' } } } + expect(coerceArgsToSchema({ tags: ['a'] }, schema).args).toEqual({ tags: ['a'] }) + for (const bad of [{ a: 1 }, '$var:u/admin/watchlist']) { + const { args, clearedKeys } = coerceArgsToSchema({ tags: bad }, schema) + expect(args).toEqual({}) + expect(clearedKeys).toEqual(['tags']) + } + }) + + // Below the top the form has the same limitations as everywhere else in the product, + // and descending means resolving `oneOf` branches — where being wrong rewrites what the + // user typed into the branch they did open. + it('leaves nested and container values to the widget that renders them', () => { + const schema = { + properties: { + obj: { type: 'object', properties: { known: { type: 'string' } } }, + rows: { type: 'array', items: { type: 'object' } } + } + } + const { args, clearedKeys } = coerceArgsToSchema( + { obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }, + schema + ) + expect(args).toEqual({ obj: { known: 1, extra: 'b' }, rows: { id: 'x' } }) + expect(clearedKeys).toEqual([]) + }) + + // Both sides parsed, never written as literals: `__proto__:` in an object literal is + // the prototype setter, so a literal declares nothing to coerce in the first place. + it('keeps a declared __proto__ instead of losing it to the setter', () => { + const { args } = coerceArgsToSchema( + JSON.parse('{"__proto__":"legit","keep":1}'), + JSON.parse('{"properties":{"__proto__":{"type":"string"},"keep":{"type":"number"}}}') + ) + expect(Object.hasOwn(args, '__proto__')).toBe(true) + expect(args['__proto__']).toBe('legit') + }) +}) + +describe('enforceDisabledDefaults', () => { + const schema = { + properties: { + locked: { type: 'string', disabled: true, default: 'fixed' }, + open: { type: 'string' } + } + } + + it('overwrites a disabled field and reports only what it changed', () => { + expect(enforceDisabledDefaults({ locked: 'mine', open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: ['locked'] + }) + // Never supplied is not overwritten: the field shows the default either way, and a + // caller told otherwise would try to correct what it never sent. + expect(enforceDisabledDefaults({ open: 'ok' }, schema)).toEqual({ + args: { locked: 'fixed', open: 'ok' }, + resetKeys: [] + }) + }) + + it('reports no reset for an object default the caller already matched', () => { + const objSchema = { + properties: { conf: { type: 'object', disabled: true, default: { a: 1 } } } + } + expect(enforceDisabledDefaults({ conf: { a: 1 } }, objSchema).resetKeys).toEqual([]) + }) +}) + +describe('secret args at every level the form nests', () => { + const schema = { + properties: { + top: { type: 'string', password: true }, + obj: { properties: { inner: { type: 'string', password: true } } }, + list: { items: { properties: { secret: { type: 'string', password: true } } } }, + either: { + oneOf: [ + { title: 'a', properties: { key: { type: 'string', password: true } } }, + { title: 'b', properties: { other: { type: 'string', password: true } } } + ] + } + } + } + const args = { + top: 'hunter2', + obj: { inner: '$var:u/ada/prod', keep: 1 }, + list: [{ secret: 'one', name: 'a' }, { secret: 'two' }], + // Tagged as branch 'a', but 'b' is stripped too: the tag is runtime state. + either: { kind: 'a', key: 'k', other: 'o' } + } + + it('redacts every value and keeps every reference', () => { + const redacted = JSON.stringify(redactSecretArgs(args, schema)) + for (const secret of ['hunter2', 'one', 'two', '"k"', '"o"']) { + expect(redacted).not.toContain(secret) + } + expect(redacted).toContain('') + expect(redacted).toContain('"name":"a"') + expect(redacted).toContain('$var:u/ada/prod') + }) + + // ArgInput synthesises '' for every untouched string, so marking one would put a hidden + // value on the card for a field nobody filled in — and mint nothing to back it. + it('leaves an empty secret empty', () => { + expect( + redactSecretArgs({ tok: '' }, { properties: { tok: { type: 'string', password: true } } }) + ).toEqual({ tok: '' }) + }) + + it('reaches a secret under a oneOf branch of an array element', () => { + const oneOfItems = { + properties: { + steps: { + type: 'array', + items: { + oneOf: [{ title: 'push', properties: { token: { type: 'string', password: true } } }] + } + } + } + } + expect(redactSecretArgs({ steps: [{ token: 'hunter2', name: 'a' }] }, oneOfItems)).toEqual({ + steps: [{ token: '', name: 'a' }] + }) + }) + + // The walk descends on the value's shape: routing an array down `properties` because the + // declaration carries that key would visit none of its elements, leaving the secret in + // the persisted card verbatim. + it('reaches through a declaration carrying both items and properties', () => { + const both = { + properties: { + creds: { + type: 'array', + items: { properties: { token: { type: 'string', password: true } } }, + properties: { token: { type: 'string', password: true } } + } + } + } + expect(redactSecretArgs({ creds: [{ token: 'hunter2' }] }, both)).toEqual({ + creds: [{ token: '' }] + }) + }) + + // A container shaped unlike its declaration is kept, so the walk has to reach in through + // the half the declaration does carry — on the value's shape alone it stops at the + // mismatch, leaving the secret there for the persisted card and the model to read. + it('reaches through a container shaped unlike its declaration', () => { + const declaresArray = { + properties: { + rows: { type: 'array', items: { properties: { token: { password: true } } } } + } + } + expect(redactSecretArgs({ rows: { token: 'hunter2' } }, declaresArray)).toEqual({ + rows: { token: '' } + }) + + const declaresObject = { + properties: { cfg: { type: 'object', properties: { token: { password: true } } } } + } + expect(redactSecretArgs({ cfg: [{ token: 'hunter2' }] }, declaresObject)).toEqual({ + cfg: [{ token: '' }] + }) + }) +}) + +describe('redactFileArgs', () => { + const schema = { + properties: { + doc: { type: 'string', contentEncoding: 'base64' }, + pics: { type: 'array', items: { type: 'string', contentEncoding: 'base64' } }, + wrap: { properties: { inner: { type: 'string', contentEncoding: 'base64' } } }, + note: { type: 'string' } + } + } + + it('replaces the bytes with a size marker at every level, and keeps the rest', () => { + const oneMeg = 'A'.repeat(1024 * 1024 * 2) + const redacted = redactFileArgs( + { doc: oneMeg, pics: ['B'.repeat(4096)], wrap: { inner: 'C'.repeat(2048) }, note: 'hi' }, + schema + ) + expect(redacted).toEqual({ + doc: '', + pics: [''], + wrap: { inner: '' }, + note: 'hi' + }) + }) +}) diff --git a/frontend/src/lib/components/job_args.ts b/frontend/src/lib/components/job_args.ts index ef3377081f..9698c03cd4 100644 --- a/frontend/src/lib/components/job_args.ts +++ b/frontend/src/lib/components/job_args.ts @@ -1,5 +1,303 @@ +/** + * A job's arguments prepared for a run form, its readers, and a result view. Coercing must + * not lose what the caller meant to send, so it is exact and shallow; stripping and + * redacting only blank a field, so they go to any depth and err towards visiting too much. + */ import { deepEqual } from 'fast-equals' +const isLockedProp = (prop: any) => !!prop?.disabled && 'default' in prop + +/** + * A field the schema disables is not the caller's to set: the run sends the schema's + * default whatever it holds. Top-level only, like every filter here. Returns the keys it + * overwrote; notifying is the caller's job. + */ +export function enforceDisabledDefaults( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; resetKeys: string[] } { + // Null prototype: assigning a declared `__proto__` into a plain `{}` reaches the + // inherited setter and the default vanishes. Always copied — callers bind the result to + // a form that edits in place, so returning the input would write through to theirs. + const result: Record = Object.assign(Object.create(null), args) + if (!schema?.properties) return { args: { ...result }, resetKeys: [] } + const resetKeys: string[] = [] + for (const [key, prop] of Object.entries(schema.properties)) { + if (!isLockedProp(prop)) continue + // Never supplied is not overwritten, and compared by value: a default can be an + // object, where identity would report every correct run as overridden. + if (result[key] !== undefined && !deepEqual(result[key], prop.default)) resetKeys.push(key) + result[key] = prop.default + } + return { args: { ...result }, resetKeys } +} + +/** How a form says what {@link enforceDisabledDefaults} overwrote, shared by the two that + * run it so the wording cannot drift apart. */ +export const resetKeysToast = (resetKeys: string[]): string => + `Disabled field${resetKeys.length > 1 ? 's' : ''} ${resetKeys + .map((k) => `'${k}'`) + .join(', ')} reset to default value${resetKeys.length > 1 ? 's' : ''}` + +/** Types `setInputCat` routes to a widget bound to a scalar. */ +const SCALAR_TYPES = new Set(['string', 'number', 'integer', 'boolean']) + +/** + * Declares an array though its `type` says `object`. A mismatch here throws rather than + * reading wrong: `MultiSelect` maps over the value as it renders, so anything else takes + * the form down, Cancel with it — a reference included, since it draws before resolving. + */ +const declaresDynMultiselect = (prop: any) => + typeof prop?.format === 'string' && prop.format.startsWith('dynmultiselect-') + +const fitsScalarType = (value: any, type: string): boolean => + type === 'integer' ? typeof value === 'number' : typeof value === type + +/** + * Resolved at run time, so the declared type describes what the job receives and never the + * string standing in for it. `ArgInput.validateInput` blesses these ahead of every type check. + */ +const REFERENCE_PREFIXES = ['$var:', '$res:', '$jsonvar:'] +const isReference = (value: any): boolean => + typeof value === 'string' && REFERENCE_PREFIXES.some((prefix) => value.startsWith(prefix)) + +/** No plain reading in the declared type; distinct from a value that reads as `undefined`. */ +const UNCOERCIBLE = Symbol('uncoercible') + +/** + * The value a scalar widget would stand for, or {@link UNCOERCIBLE}. Only conversions with + * one plain reading: a number input shows `"7"` as 7 and a toggle shows any non-empty + * string as on, so guessing past this would put a value on screen that nobody wrote. + */ +function coerceScalar(value: any, type: string): any { + if (typeof value === 'object') return UNCOERCIBLE + if (type === 'string') { + return typeof value === 'number' || typeof value === 'boolean' ? String(value) : UNCOERCIBLE + } + if (typeof value !== 'string') return UNCOERCIBLE + const trimmed = value.trim() + if (type === 'number' || type === 'integer') { + if (trimmed === '') return UNCOERCIBLE + const parsed = Number(trimmed) + return Number.isFinite(parsed) ? parsed : UNCOERCIBLE + } + if (type === 'boolean') { + if (trimmed.toLowerCase() === 'true') return true + if (trimmed.toLowerCase() === 'false') return false + } + return UNCOERCIBLE +} + +/** + * Drop every argument the schema does not declare, naming them. The schema is what every run + * surface builds its fields from, so a value under a name it never declares has no widget + * anywhere: sending one is sending what nobody could see or edit before the run. + */ +export function dropUndeclaredArgs( + args: Record, + schema: { properties?: Record } | undefined +): { args: Record; undeclaredKeys: string[] } { + const properties = schema?.properties ?? {} + // hasOwn, not `in`: every object inherits `constructor` and `toString`, so `in` would + // hand an inherited declaration to an argument the schema never named. + const kept: Record = Object.create(null) + const undeclaredKeys: string[] = [] + for (const [key, value] of Object.entries(args ?? {})) { + if (Object.hasOwn(properties, key)) kept[key] = value + else undeclaredKeys.push(key) + } + return { args: { ...kept }, undeclaredKeys } +} + +/** + * Make arguments say what the run form will show, then apply {@link enforceDisabledDefaults}. + * A scalar widget renders its own reading of a wrong-typed value and never writes it back, so + * an untouched form would submit what it never displayed; a value with no reading is cleared. + * Top-level only: descending means resolving `oneOf`, where being wrong rewrites user input. + */ +export function coerceArgsToSchema( + args: Record, + schema: { properties?: Record } | undefined +): { + args: Record + resetKeys: string[] + clearedKeys: string[] + undeclaredKeys: string[] +} { + const properties = schema?.properties ?? {} + const clearedKeys: string[] = [] + const { args: declared, undeclaredKeys } = dropUndeclaredArgs(args, schema) + const kept: Record = Object.create(null) + for (const [key, value] of Object.entries(declared)) { + // Declared, but a declaration can still be nothing, and reading `.type` off it throws. + const prop = properties[key] + if ( + prop === undefined || + value == null || + (isReference(value) && !declaresDynMultiselect(prop)) + ) { + kept[key] = value + continue + } + if (declaresDynMultiselect(prop)) { + if (Array.isArray(value)) kept[key] = value + else clearedKeys.push(key) + continue + } + if (!SCALAR_TYPES.has(prop.type) || fitsScalarType(value, prop.type)) { + kept[key] = value + continue + } + const coerced = coerceScalar(value, prop.type) + if (coerced === UNCOERCIBLE) clearedKeys.push(key) + else kept[key] = coerced + } + const { args: result, resetKeys } = enforceDisabledDefaults({ ...kept }, schema) + return { args: result, resetKeys, clearedKeys, undeclaredKeys } +} + +/** + * Every bag of `properties` a declaration can show a value's keys through, including every + * `oneOf` branch rather than the selected one: a secret under a variant nobody opened + * leaves the form just the same. + */ +function declarationBags(prop: any): Record[] { + const bags: Record[] = [] + if (prop?.properties) bags.push(prop.properties) + if (Array.isArray(prop?.oneOf)) + for (const branch of prop.oneOf) if (branch?.properties) bags.push(branch.properties) + return bags +} + +/** + * Apply `visit` to every value whose declaration matches `isLeaf`, at any depth; returning + * `undefined` removes it. Recursive because the form is, so a level left unvisited is one a + * secret can sit at. Descends on the value's shape, never on the declaration's keys: one + * carrying both `items` and `properties` must not route a shape down the other's branch. + */ +function mapLeaves( + value: any, + prop: any, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown, + path: (string | number)[] +): any { + if (value == null || typeof value !== 'object') return value + // A container shaped unlike its declaration is kept rather than dropped, since the widget + // is the one that reports it — so the walk has to reach in through whichever half the + // declaration does carry, or a secret under one leaves the form verbatim. + if (Array.isArray(value)) + return value.map((item, i) => mapLeaves(item, prop?.items ?? prop, isLeaf, visit, [...path, i])) + const bags = declarationBags(prop) + if (bags.length === 0) + return prop?.items ? mapLeaves(value, prop.items, isLeaf, visit, path) : value + // Null prototype, and keyed off the value rather than the declaration: a key is only + // ever rewritten where it already exists, so no branch of a `oneOf` can add one. + const result: Record = Object.assign(Object.create(null), value) + for (const key of Object.keys(result)) { + const declared = bags.filter((bag) => Object.hasOwn(bag, key)).map((bag) => bag[key]) + // Segments, never a joined name: a key can itself hold a dot, and two leaves reported + // under one name let a caller correlating by it take the one for the other. + const keyPath = [...path, key] + // A matching object is a leaf, not a level: a password object is stored whole as a + // single $jsonvar: reference, and a file is one opaque base64 string. + const leaf = declared.find(isLeaf) + if (leaf) { + const mapped = visit(result[key], leaf, keyPath) + if (mapped === undefined) delete result[key] + else result[key] = mapped + continue + } + for (const declaration of declared) + result[key] = mapLeaves(result[key], declaration, isLeaf, visit, keyPath) + } + return { ...result } +} + +export const isSecretProp = (prop: any) => !!prop?.password + +const isFileProp = (prop: any) => + prop?.contentEncoding === 'base64' || prop?.items?.contentEncoding === 'base64' + +function fileMarker(base64: string): string { + const bytes = Math.floor((base64.length * 3) / 4) + return bytes < 1024 * 1024 + ? `` + : `` +} + +/** {@link mapLeaves} over a whole argument object, against a schema that may declare + * nothing to match. Copies either way, for the reason {@link enforceDisabledDefaults} + * copies. */ +export function mapArgLeaves( + args: Record, + schema: { properties?: Record } | undefined, + isLeaf: (prop: any) => boolean, + visit: (value: unknown, prop: any, path: (string | number)[]) => unknown +): Record { + return mapLeaves(args ?? {}, { properties: schema?.properties ?? {} }, isLeaf, visit, []) +} + +/** A leaf's path as the lines naming it to a reader read: `creds[0].secret`. */ +const formatArgPath = (path: (string | number)[]): string => + path.reduce( + (acc, segment) => + typeof segment === 'number' + ? `${acc}[${segment}]` + : acc + ? `${acc}.${segment}` + : String(segment), + '' + ) + +/** + * Drop every file argument, so a caller cannot propose file bytes on the user's behalf: + * the field opens empty and the user attaches the file. Bytes a form is prefilled with + * are bytes the stored transcript carries, unbounded, for a value no caller can produce. + * Reports the path of each one removed, or the caller reads the absence as the user having + * deleted the value. + */ +export function stripFileArgs( + args: Record, + schema: { properties?: Record } | undefined, + strippedKeys?: string[] +): Record { + return mapArgLeaves(args, schema, isFileProp, (value, _prop, path) => { + if (value !== undefined) strippedKeys?.push(formatArgPath(path)) + return undefined + }) +} + +/** + * Replace a sensitive value with a fixed marker, for text that leaves the form. A reference is + * kept: it names a variable rather than holding one, and the run page shows the same job's + * arguments that way. An empty field is kept for the reason `processSecretArgs` mints nothing + * for one — marking it would describe a secret the run never carried. + */ +export function redactSecretArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + return mapArgLeaves(args, schema, isSecretProp, (value) => + value == null ? undefined : value === '' || isReference(value) ? value : '' + ) +} + +/** + * Replace every file argument with a marker naming its size. The base64 belongs in the + * job request and nowhere else: rendered it is unreadable, persisted it is unbounded, and + * a file small enough to survive truncation reaches the model whole. + */ +export function redactFileArgs( + args: Record, + schema: { properties?: Record } | undefined +): Record { + const mark = (value: unknown) => (typeof value === 'string' ? fileMarker(value) : value) + return mapArgLeaves(args, schema, isFileProp, (value) => + Array.isArray(value) ? value.map(mark) : mark(value) + ) +} + export function isWindmillTooBigObject(obj: any): boolean { return ( typeof obj === 'object' && diff --git a/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte b/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte index 6d996e76e4..c6b86b1cc7 100644 --- a/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte +++ b/frontend/src/lib/components/jobs/MissingWorkerTagAlert.svelte @@ -2,9 +2,11 @@ import { base } from '$app/paths' import Alert from '$lib/components/common/alert/Alert.svelte' import { WorkerService } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { resource } from 'runed' import { hasWorkerForTag } from './missingWorker' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { /** Worker tag the feature's jobs run on. */ @@ -17,7 +19,7 @@ let { tag, subject, workspace = undefined, class: className = '' }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) // With per-workspace default tags, jobs run on a workspace-suffixed variant of // `tag`, so probing the bare tag would warn about a tag nothing uses. The diff --git a/frontend/src/lib/components/markdownProse.ts b/frontend/src/lib/components/markdownProse.ts index bc943aa42b..3f79ab79a5 100644 --- a/frontend/src/lib/components/markdownProse.ts +++ b/frontend/src/lib/components/markdownProse.ts @@ -4,10 +4,14 @@ * call site with layout-only classes (padding, width, bg); anything typographic * belongs here. * - * - 'xs': micro scale for dense secondary panes (chat reasoning blocks) + * - 'xs': micro scale for dense secondary panes (chat reasoning blocks, group notes) * - 'sm': compact chat-bubble scale (assistant messages, flow/app chat, settings) * - 'doc': same rhythm and body size as 'sm', with a taller heading ramp * (lg/base/sm) and semibold headings for document-like surfaces (artifacts) + * + * h1 and h2 step above the body size in every preset, so `#` and `##` read as + * headings rather than bold body text. Below that the tight 'xs' and 'sm' scales + * run out of room, and h3 down differentiates by weight and colour alone. */ // Kept as literal template parts: Tailwind's scanner reads class names verbatim @@ -22,14 +26,14 @@ const base = // One vertical rhythm for sm/doc; heading margins stay per-preset (fixed, not // the plugin's em-based ones) so 'doc' can breathe more between sections. -const rhythm = 'prose-sm leading-snug prose-ul:!pl-6' +const rhythm = 'prose-sm leading-snug' const bodyXs = 'text-primary prose-p:text-primary prose-li:text-primary prose-p:text-xs prose-li:text-xs prose-code:text-xs prose-pre:text-xs prose-table:text-xs' export const markdownProse = { - xs: `${base} prose-sm leading-snug prose-ul:!pl-5 prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-2xs prose-h2:text-2xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, - sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-sm prose-h2:text-xs prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, + xs: `${base} prose-sm leading-snug prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-sm prose-h2:text-xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, + sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-base prose-h2:text-sm prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, doc: `${base} ${rhythm} ${bodyXs} prose-headings:mt-8 prose-headings:mb-2 prose-headings:font-semibold prose-headings:text-emphasis prose-h1:text-lg prose-h2:text-base prose-h3:text-sm prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs prose-pre:bg-transparent prose-pre:p-0` } as const diff --git a/frontend/src/lib/components/mcp/McpConnect.svelte b/frontend/src/lib/components/mcp/McpConnect.svelte index 19b4d32209..1fe1a2fed9 100644 --- a/frontend/src/lib/components/mcp/McpConnect.svelte +++ b/frontend/src/lib/components/mcp/McpConnect.svelte @@ -27,9 +27,12 @@ /** Required: a caller that forgot it would create the connection in whichever * workspace the ui happens to be showing, not the one it operates on. */ workspace: string + /** Off where the surface around it already draws a card — a popover panel — + * so the two do not stack a border and a background on each other. */ + bordered?: boolean } - let { onConnected, onCancel, workspace }: Props = $props() + let { onConnected, onCancel, workspace, bordered = true }: Props = $props() let ws = $derived(workspace) // Any URL is connectable; a suggestion is a shortcut that also pins how the @@ -130,13 +133,6 @@ let canSignIn = $derived( oauthAppReady || (canDiscover && !!$enterpriseLicense && discoveryFoundOAuth !== false) ) - // The action button names the credential, not the outcome, so the path field - // says what clicking it will leave behind. - let pathNote = $derived( - canSignIn && !showToken - ? 'Signing in saves the connection at this path, as an' - : 'The connection is saved at this path, as an' - ) // Why the token field is the only way in, said where the token is asked for. let tokenNote = $derived( needsOauthApp && entry @@ -338,7 +334,11 @@ } -
    +
    Connect an MCP server {#if onCancel} @@ -442,8 +442,12 @@ {/if}
    diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index bf26605ac2..b88907bf02 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -19,6 +19,7 @@ } from './previewRouter' import { withMenuHidden } from './sessionMode.svelte' import ArtifactViewer from '../copilot/chat/artifacts/ArtifactViewer.svelte' + import RunFormPreviewSlot from './RunFormPreviewSlot.svelte' import { setOverlayHost } from '../common/overlayHost.svelte' let { @@ -102,12 +103,19 @@ applyPageIframeTheme(darkMode) }) + // A page item's editor reads its draft only when it loads, so a reload remounts it. + let pageItemReloadNonce = $state(0) + export function reload() { // A live editor shares the runtime store the chat mutates, so generic chat // edits are already reflected — no reload needed. Deploys refresh it via // each editor view's onDeploy → runtime.syncPreviewWithDeployed. So only the // iframe fallback (a separate page) has to be told to refresh. if (slot.kind === 'editor') return + if (slot.kind === 'pageitem') { + pageItemReloadNonce++ + return + } try { const win = frame?.contentWindow if (!win) return @@ -310,6 +318,22 @@ {/await} {/if}
    +{:else if slot.kind === 'pageitem' && mounted && runtime} +
    + + {#if overlayHostEl} + {#await import('./PageItemEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} + {/if} +
    {:else if slot.kind === 'artifact' && mounted}
    This artifact is no longer available.
    {/if}
    +{:else if slot.kind === 'runform' && mounted} +
    + + {#if runtime && overlayHostEl} + + {/if} +
    {:else if mounted}
    + {#if item.leadsCatalogue} +

    + More from the hub +

    + {/if} + +